From 6c8b5f85d2c679af78b09f1d53047de54e51efe6 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:23:49 +0530 Subject: [PATCH 01/10] added unit testing --- pytest.ini | 2 + requirements.txt | 1 + tests/TEST_CASES.csv | 156 +-- tests/conftest.py | 336 ++++++ tests/test_kafka_events.py | 71 -- tests/test_mode_logic.py | 177 --- tests/unit_testing.py | 2220 ++++++++++++++++++++++++++++++++++++ 7 files changed, 2637 insertions(+), 326 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/conftest.py delete mode 100644 tests/test_kafka_events.py delete mode 100644 tests/test_mode_logic.py create mode 100644 tests/unit_testing.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..19ef46d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = test_*.py unit_testing.py diff --git a/requirements.txt b/requirements.txt index 2a47430..c72986c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ fastapi>=0.112.0 python-multipart>=0.0.9 uvicorn>=0.30.0 pytest>=8.0.0 +httpx>=0.24.0 json5>=0.14.0 streamlit>=1.35.0 pandas>=2.0.0 diff --git a/tests/TEST_CASES.csv b/tests/TEST_CASES.csv index 1dd30d3..4da7011 100644 --- a/tests/TEST_CASES.csv +++ b/tests/TEST_CASES.csv @@ -7,7 +7,7 @@ KAFKA-002,Kafka Ingestion,Valid story CREATE event ingested successfully,Positiv 2) DB: SELECT * FROM submissions WHERE submission_id='3280' AND tenant_code='mitra'; -> row exists, status is 'pending'/'processing'/'success' 3) DB: SELECT objective, challenge, transcript_link FROM story_submissions WHERE submission_id='3280' AND tenant_code='mitra'; -> fields populated matching the fixture 4) Confirm no DLQ message for submissionId 3280" -KAFKA-003,Kafka Ingestion,Valid UPDATE event applies delta-only newValues,Positive,Critical,An existing submission from a prior CREATE (run KAFKA-001 first),"Publish an UPDATE event with newValues containing only a subset of fields (e.g. title, participantsData)","Only the fields present in newValues change; all other columns (e.g. challenges, solutions) remain untouched via COALESCE",Not Tested,"1) DB (before): SELECT title, challenges, solutions FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; -> note the values +KAFKA-003,Kafka Ingestion,Valid UPDATE event applies delta-only newValues,Positive,Critical,An existing submission from a prior CREATE (run KAFKA-001 first),"Publish an UPDATE event with newValues containing only a subset of fields (e.g. title, participantsData)","Only the fields present in newValues change; all other columns (e.g. challenges, solutions) remain untouched via COALESCE",Verified,"1) DB (before): SELECT title, challenges, solutions FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; -> note the values 2) Run: python tests/kafka_push.py --file tests/kafka_events/update/update_discussion.json 3) DB (after): re-run the same SELECT -> title/participantsData reflect the new values from newValues; challenges/solutions are UNCHANGED from step 1 4) Confirm no DLQ message for this submissionId" @@ -15,10 +15,10 @@ KAFKA-004,Kafka Ingestion,Valid DELETE event removes submission,Positive,Critica 2) DB: SELECT * FROM submissions WHERE submission_id='200' AND tenant_code='mitra'; -> expect 0 rows 3) DB: SELECT * FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; -> expect 0 rows (ON DELETE CASCADE) 4) DB: SELECT * FROM analysis_results WHERE submission_id='200' AND tenant_code='mitra'; -> expect 0 rows" -KAFKA-005,Kafka Ingestion,Malformed JSON payload routed to DLQ,Negative,High,Consumer running,Publish a non-JSON string (e.g. truncated/corrupt JSON) to the ingestion topic,json.JSONDecodeError caught; message routed to analytics.ingestion.raw.dlq with reason 'Invalid JSON: ...'; original topic offset committed,Not Tested,"1) Use a raw producer (kafka-console-producer.sh or a short Python snippet with confluent_kafka.Producer) to publish the literal text '{bad json' (not valid JSON) to analytics.ingestion.raw +KAFKA-005,Kafka Ingestion,Malformed JSON payload routed to DLQ,Negative,High,Consumer running,Publish a non-JSON string (e.g. truncated/corrupt JSON) to the ingestion topic,json.JSONDecodeError caught; message routed to analytics.ingestion.raw.dlq with reason 'Invalid JSON: ...'; original topic offset committed,Verified,"1) Use a raw producer (kafka-console-producer.sh or a short Python snippet with confluent_kafka.Producer) to publish the literal text '{bad json' (not valid JSON) to analytics.ingestion.raw 2) Consume analytics.ingestion.raw.dlq from the earliest offset (see 'Reading the DLQ' above) and locate the message by its submissionId header or by the submissionId inside the payload. reason header should start with 'Invalid JSON:' 3) DB: confirm no new submissions row was created for this message" -KAFKA-006,Kafka Ingestion,Non-object JSON payload routed to DLQ,Negative,High,Consumer running,"Publish a valid JSON array or scalar (e.g. ""[]"" or ""42"") instead of an object","Routed to DLQ with reason 'Expected a JSON object, got list/int'",Not Tested,"1) Publish the literal text '[]' (valid JSON, but not an object) to analytics.ingestion.raw +KAFKA-006,Kafka Ingestion,Non-object JSON payload routed to DLQ,Negative,High,Consumer running,"Publish a valid JSON array or scalar (e.g. ""[]"" or ""42"") instead of an object","Routed to DLQ with reason 'Expected a JSON object, got list/int'",Verified,"1) Publish the literal text '[]' (valid JSON, but not an object) to analytics.ingestion.raw 2) Consume analytics.ingestion.raw.dlq from the earliest offset (see 'Reading the DLQ' above) and locate the message by its submissionId header or by the submissionId inside the payload. reason header should read 'Expected a JSON object, got list' 3) Repeat with the literal text '42' -> reason should read '...got int'" KAFKA-007,Kafka Ingestion,Missing submissionId on CREATE routed to DLQ,Negative,Critical,Consumer running,Publish a CREATE event with the submissionId key entirely absent,"_validate_ingestion_schema reports 'submissionId' is missing; message routed to DLQ, never reaches insert_or_update_submission",Verified,"1) Copy tests/kafka_events/create/create_discussion.json, delete the ""submissionId"" key, save as /tmp/no_submission_id.json, push it: python tests/kafka_push.py --file /tmp/no_submission_id.json @@ -27,7 +27,7 @@ KAFKA-007,Kafka Ingestion,Missing submissionId on CREATE routed to DLQ,Negative, KAFKA-008,Kafka Ingestion,Null submissionId routed to DLQ (not coerced to truthy string),Negative,Critical,Consumer running,Publish a CREATE event with submissionId: null,"Correctly detected as null (not silently converted to the truthy string ""None""); routed to DLQ",Verified,"1) Copy a create fixture, set ""submissionId"": null, push it 2) Consume analytics.ingestion.raw.dlq from the earliest offset (see 'Reading the DLQ' above) and locate the message by its submissionId header or by the submissionId inside the payload. reason should include ""'submissionId' is null"" 3) DB: SELECT * FROM submissions WHERE submission_id='None'; -> MUST return 0 rows (this is the specific regression this case guards against)" -KAFKA-009,Kafka Ingestion,Missing tenantCode routed to DLQ,Negative,Critical,Consumer running,Publish an event with tenantCode absent,Routed to DLQ with reason citing 'tenantCode' is missing,Not Tested,"1) Copy a create fixture, delete the ""tenantCode"" key, push it +KAFKA-009,Kafka Ingestion,Missing tenantCode routed to DLQ,Negative,Critical,Consumer running,Publish an event with tenantCode absent,Routed to DLQ with reason citing 'tenantCode' is missing,Verified,"1) Copy a create fixture, delete the ""tenantCode"" key, push it 2) Consume analytics.ingestion.raw.dlq from the earliest offset (see 'Reading the DLQ' above) and locate the message by its submissionId header or by the submissionId inside the payload. reason should include ""'tenantCode' is missing"" 3) DB: confirm no submissions row was created for that submissionId" KAFKA-010,Kafka Ingestion,Empty tags.state on discussion/story CREATE routed to DLQ,Negative,High,Consumer running,"Publish a CREATE event with tags.state = """"",Routed to DLQ with reason 'tags.state is empty',Verified,"1) Copy a create fixture, set tags.state to """", push it @@ -54,10 +54,10 @@ KAFKA-016,Kafka Ingestion,Unsupported eventType routed to DLQ,Negative,Medium,Co KAFKA-017,Kafka Ingestion,"UPDATE with newValues. = """" routed to DLQ",Negative,High,An existing submission,"Publish an UPDATE event with newValues.title = """"",newValuesNoEmpty check catches it; routed to DLQ with reason 'newValues.title is empty',Verified,"1) Copy update_discussion.json, set newValues.title to """", push it 2) Consume analytics.ingestion.raw.dlq from the earliest offset (see 'Reading the DLQ' above) and locate the message by its submissionId header or by the submissionId inside the payload. reason should include ""'newValues.title' is empty"" 3) DB: confirm the submission's title in discussion_submissions is UNCHANGED from before the push" -KAFKA-018,Kafka Ingestion,UPDATE omitting a field from newValues preserves the existing DB value,Edge,High,An existing submission with populated challenges/solutions,Publish an UPDATE event whose newValues omits challenges/solutions entirely,Those columns remain unchanged in the DB (COALESCE preserves them); validator does not flag the omission as an error,Not Tested,"1) DB (before): SELECT challenges, solutions FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; +KAFKA-018,Kafka Ingestion,UPDATE omitting a field from newValues preserves the existing DB value,Edge,High,An existing submission with populated challenges/solutions,Publish an UPDATE event whose newValues omits challenges/solutions entirely,Those columns remain unchanged in the DB (COALESCE preserves them); validator does not flag the omission as an error,Verified,"1) DB (before): SELECT challenges, solutions FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; 2) Push update_discussion.json unmodified (its newValues only has title/participantsData, no challenges/solutions) 3) DB (after): re-run the same SELECT -> challenges/solutions identical to step 1; no DLQ message" -KAFKA-019,Kafka Ingestion,Duplicate CREATE for an existing submissionId is skipped,Edge,Medium,A submission already exists for submission_id + tenant_code,Publish a second CREATE event with the same submissionId/tenantCode,"Consumer detects the existing row, logs a warning, and skips re-insertion (no duplicate row, no crash)",Not Tested,"1) Push create_story.json once (submission 3280 created) +KAFKA-019,Kafka Ingestion,Duplicate CREATE for an existing submissionId is skipped,Edge,Medium,A submission already exists for submission_id + tenant_code,Publish a second CREATE event with the same submissionId/tenantCode,"Consumer detects the existing row, logs a warning, and skips re-insertion (no duplicate row, no crash)",Verified,"1) Push create_story.json once (submission 3280 created) 2) Push the exact same file again 3) DB: SELECT count(*) FROM submissions WHERE submission_id='3280' AND tenant_code='mitra'; -> must be exactly 1 4) App log should contain a 'Duplicate entry' warning for submission 3280" @@ -92,51 +92,51 @@ SEC-004,Security & Logging,Thematic classification LLM response never logged in CONFIG-001,Config & Settings,Valid PROCESS_CONFIG_STORY/DISCUSSION JSON loads at startup,Positive,High,Well-formed JSON in both env vars,Start the app / import app.config.settings,settings.get_process_config() returns the parsed step list for 'story' and 'discussion',Verified,"1) Start the app normally (python main.py --mode all or your usual entrypoint) 2) Confirm it starts without a pydantic ValidationError 3) Push a create fixture and confirm the expected pipeline steps run (visible as 'Starting workflow step' log lines matching PROCESS_CONFIG_STORY/DISCUSSION)" -CONFIG-002,Config & Settings,Invalid JSON in PROCESS_CONFIG_STORY fails fast at startup,Negative,High,PROCESS_CONFIG_STORY set to invalid JSON,Start the app,pydantic field_validator raises at settings construction time instead of silently defaulting to [] and skipping all story processing,Not Tested,"1) In a test .env, set PROCESS_CONFIG_STORY=not valid json +CONFIG-002,Config & Settings,Invalid JSON in PROCESS_CONFIG_STORY fails fast at startup,Negative,High,PROCESS_CONFIG_STORY set to invalid JSON,Start the app,pydantic field_validator raises at settings construction time instead of silently defaulting to [] and skipping all story processing,Verified,"1) In a test .env, set PROCESS_CONFIG_STORY=not valid json 2) Start the app 3) Confirm it FAILS to start with a clear 'Invalid JSON configuration for PROCESS_CONFIG_STORY' error, rather than starting silently" CONFIG-003,Config & Settings,Valid STORY_KAFKA_SCHEMA/DISCUSSION_KAFKA_SCHEMA loads correctly,Positive,High,Well-formed JSON in both env vars,Call settings.get_kafka_ingestion_schema('story') and ('discussion'),Returns a dict with 'create'/'update'/'delete' keys matching the configured schema,Verified,"1) python3 -c ""from app.config import settings; print(settings.get_kafka_ingestion_schema('story').keys())"" 2) Confirm output is dict_keys(['create', 'update', 'delete']) 3) Repeat for 'discussion'" -CONFIG-004,Config & Settings,Malformed Kafka ingestion schema (missing a required top-level key) fails validation,Negative,High,STORY_KAFKA_SCHEMA missing the 'delete' key,Start the app,validate_kafka_ingestion_schema_json raises ValueError at startup,Not Tested,"1) In a test .env, edit STORY_KAFKA_SCHEMA to remove the ""delete"": {...} section +CONFIG-004,Config & Settings,Malformed Kafka ingestion schema (missing a required top-level key) fails validation,Negative,High,STORY_KAFKA_SCHEMA missing the 'delete' key,Start the app,validate_kafka_ingestion_schema_json raises ValueError at startup,Verified,"1) In a test .env, edit STORY_KAFKA_SCHEMA to remove the ""delete"": {...} section 2) Start the app 3) Confirm it FAILS to start with ""must be a JSON object with 'create', 'update', and 'delete' keys""" -CONFIG-005,Config & Settings,Malformed schema section (required not a list of strings) fails validation,Edge,Medium,A schema section's 'required' field set to a string instead of a list,Start the app,Validator raises ValueError instead of letting the consumer crash later at message-processing time,Not Tested,"1) In a test .env, edit DISCUSSION_KAFKA_SCHEMA so create.required is a plain string instead of a list +CONFIG-005,Config & Settings,Malformed schema section (required not a list of strings) fails validation,Edge,Medium,A schema section's 'required' field set to a string instead of a list,Start the app,Validator raises ValueError instead of letting the consumer crash later at message-processing time,Verified,"1) In a test .env, edit DISCUSSION_KAFKA_SCHEMA so create.required is a plain string instead of a list 2) Start the app 3) Confirm it FAILS to start with '...must be a list of strings' rather than starting and crashing later on the first message" -CONFIG-006,Config & Settings,RESET_DB=true with ENVIRONMENT=production is refused,Negative,Critical,"ENVIRONMENT=production, RESET_DB=true",Start the app / call initialize_schema(),Schema is NOT dropped; an error is logged instructing to set ENVIRONMENT=development if intentional,Not Tested,"1) In a test .env, set ENVIRONMENT=production and RESET_DB=true (use a disposable/QA DB only!) +CONFIG-006,Config & Settings,RESET_DB=true with ENVIRONMENT=production is refused,Negative,Critical,"ENVIRONMENT=production, RESET_DB=true",Start the app / call initialize_schema(),Schema is NOT dropped; an error is logged instructing to set ENVIRONMENT=development if intentional,Verified,"1) In a test .env, set ENVIRONMENT=production and RESET_DB=true (use a disposable/QA DB only!) 2) Start the app 3) DB: confirm existing tables/rows are still present (e.g. SELECT count(*) FROM submissions;) — schema was NOT dropped 4) App log should contain 'RESET_DB=True was requested but ENVIRONMENT=...production... — refusing to drop the schema'" -CONFIG-007,Config & Settings,RESET_DB=true with ENVIRONMENT=development resets the schema,Positive,Medium,"ENVIRONMENT=development, RESET_DB=true",Start the app,"public schema is dropped and recreated as intended, with a warning logged",Not Tested,"CAUTION: only run against a disposable QA database. +CONFIG-007,Config & Settings,RESET_DB=true with ENVIRONMENT=development resets the schema,Positive,Medium,"ENVIRONMENT=development, RESET_DB=true",Start the app,"public schema is dropped and recreated as intended, with a warning logged",Verified,"CAUTION: only run against a disposable QA database. 1) Set ENVIRONMENT=development, RESET_DB=true; start the app 2) DB: confirm all tables are empty/freshly created (e.g. SELECT count(*) FROM submissions; -> 0) 3) App log should contain 'Database schema reset requested; dropped and recreated public schema'" -CONFIG-008,Config & Settings,get_process_config for an unrecognized submission type returns [],Edge,Low,submission_type is neither 'story' nor 'discussion',Call settings.get_process_config('survey'),Returns an empty list rather than raising,Not Tested,"python3 -c ""from app.config import settings; print(settings.get_process_config('survey'))"" -> expect []" +CONFIG-008,Config & Settings,get_process_config for an unrecognized submission type returns [],Edge,Low,submission_type is neither 'story' nor 'discussion',Call settings.get_process_config('survey'),Returns an empty list rather than raising,Verified,"python3 -c ""from app.config import settings; print(settings.get_process_config('survey'))"" -> expect []" CONFIG-009,Config & Settings,get_kafka_ingestion_schema for an unrecognized submission type raises,Negative,Medium,"submission_type is neither 'story' nor 'discussion', or None",Call settings.get_kafka_ingestion_schema(None),Raises ValueError('No Kafka ingestion schema defined...') instead of crashing on .lower(),Verified,"python3 -c ""from app.config import settings; settings.get_kafka_ingestion_schema(None)"" -> expect a clean ValueError, not an AttributeError/traceback about .lower()" DB-001,Database Operations,Insert new discussion submission with TEXT[] challenges/solutions,Positive,Critical,Valid discussion CREATE payload,Call insert_or_update_submission with a discussion event,"challenges/solutions stored as native Postgres TEXT[] (asyncpg round-trips as a Python list, no manual encode/decode)",Verified,"1) Push create_discussion.json 2) DB: SELECT pg_typeof(challenges) FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; -> expect 'text[]' 3) DB: SELECT array_length(challenges, 1) FROM discussion_submissions WHERE submission_id='200' AND tenant_code='mitra'; -> matches the number of statements in the fixture" DB-002,Database Operations,Insert new story submission with scalar text fields,Positive,Critical,Valid story CREATE payload,Call insert_or_update_submission with a story event,objective/challenge/impact etc. stored as plain TEXT columns,Verified,"1) Push create_story.json 2) DB: SELECT pg_typeof(objective) FROM story_submissions WHERE submission_id='3280' AND tenant_code='mitra'; -> expect 'text'" -DB-003,Database Operations,_normalize_statement_list wraps a single string in a one-element list,Edge,Medium,N/A,"Call _normalize_statement_list(""a single statement"")","Returns [""a single statement""]",Not Tested,"Developer/unit-level check: python3 -c ""from app.database.operations import _normalize_statement_list as f; print(f('a single statement'))"" -> expect ['a single statement']" -DB-004,Database Operations,_normalize_statement_list returns None for a None input,Edge,Medium,N/A,Call _normalize_statement_list(None),Returns None (distinguishing genuinely-absent from empty-list),Not Tested,"python3 -c ""from app.database.operations import _normalize_statement_list as f; print(f(None))"" -> expect None" -DB-005,Database Operations,Update with delta-only newValues does not revert already-masked PII fields,Edge,Critical,A submission whose challenges/solutions have already been PII-masked,Publish an UPDATE event whose newValues does not include challenges/solutions,"The masked values already in the DB are preserved, not overwritten with the producer's original unmasked oldValues",Not Tested,"1) Push a create fixture and wait for the PII activity to complete; DB: SELECT challenges FROM discussion_submissions WHERE submission_id='' AND tenant_code='mitra'; -> note the MASKED values +DB-003,Database Operations,_normalize_statement_list wraps a single string in a one-element list,Edge,Medium,N/A,"Call _normalize_statement_list(""a single statement"")","Returns [""a single statement""]",Verified,"Developer/unit-level check: python3 -c ""from app.database.operations import _normalize_statement_list as f; print(f('a single statement'))"" -> expect ['a single statement']" +DB-004,Database Operations,_normalize_statement_list returns None for a None input,Edge,Medium,N/A,Call _normalize_statement_list(None),Returns None (distinguishing genuinely-absent from empty-list),Verified,"python3 -c ""from app.database.operations import _normalize_statement_list as f; print(f(None))"" -> expect None" +DB-005,Database Operations,Update with delta-only newValues does not revert already-masked PII fields,Edge,Critical,A submission whose challenges/solutions have already been PII-masked,Publish an UPDATE event whose newValues does not include challenges/solutions,"The masked values already in the DB are preserved, not overwritten with the producer's original unmasked oldValues",Verified,"1) Push a create fixture and wait for the PII activity to complete; DB: SELECT challenges FROM discussion_submissions WHERE submission_id='' AND tenant_code='mitra'; -> note the MASKED values 2) Push an UPDATE event for the same submission whose newValues omits challenges/solutions but whose oldValues has the ORIGINAL unmasked text 3) DB: re-run the SELECT -> challenges must still show the MASKED values from step 1, not the original unmasked oldValues" DB-006,Database Operations,Duplicate session_id on a new submissionId raises a clear error,Negative,High,An existing submission with session_id X,Attempt to insert a new submissionId reusing session_id X,UniqueViolationError caught and re-raised as a descriptive ValueError; existing row is untouched,Verified,"1) Note an existing submission's sessionId, e.g. SELECT session_id FROM submissions LIMIT 1; 2) Copy a create fixture, set a brand-new submissionId but reuse that sessionId, push it 3) Consume analytics.ingestion.raw.dlq from the earliest offset (see 'Reading the DLQ' above) and locate the message by its submissionId header or by the submissionId inside the payload. reason should mention 'is already associated with a different submission' 4) DB: confirm the ORIGINAL submission (with that session_id) is unchanged, and no new row was created for the new submissionId" -THEME-001,Thematic Classification,Statement below MINIMUM_THEME_WORD_COUNT classified Unknown/Unclear,Negative,Medium,A statement with fewer words than the configured minimum (default 5),Run thematic_classification_activity on that statement,category_type = 'Unknown/Unclear'; no LLM call made,Not Tested,"1) Push a discussion CREATE with a challenge statement of e.g. 2 words (""Too short"") +THEME-001,Thematic Classification,Statement below MINIMUM_THEME_WORD_COUNT classified Unknown/Unclear,Negative,Medium,A statement with fewer words than the configured minimum (default 5),Run thematic_classification_activity on that statement,category_type = 'Unknown/Unclear'; no LLM call made,Verified,"1) Push a discussion CREATE with a challenge statement of e.g. 2 words (""Too short"") 2) DB: SELECT category_type FROM analysis_results WHERE submission_id='' AND tenant_code='mitra' AND statements='Too short'; -> expect 'Unknown/Unclear' 3) DB: SELECT * FROM llm_logs WHERE submission_id='' AND analysis_type='thematic_classification'; -> expect no row generated for this statement (no LLM call)" -THEME-002,Thematic Classification,Garbage/spam statement classified Unknown/Unclear,Negative,Medium,A statement that is a keyboard mash or repeated single token,Run classification on it,_is_garbage_or_spam heuristics catch it; category_type = 'Unknown/Unclear',Not Tested,"1) Push a discussion CREATE with a challenge statement like ""asdf asdf asdf asdf asdf"" +THEME-002,Thematic Classification,Garbage/spam statement classified Unknown/Unclear,Negative,Medium,A statement that is a keyboard mash or repeated single token,Run classification on it,_is_garbage_or_spam heuristics catch it; category_type = 'Unknown/Unclear',Verified,"1) Push a discussion CREATE with a challenge statement like ""asdf asdf asdf asdf asdf"" 2) DB: SELECT category_type FROM analysis_results WHERE statements LIKE 'asdf%'; -> expect 'Unknown/Unclear'" -THEME-003,Thematic Classification,Statement containing a PII mask tag classified Flagged,Positive,High,A statement containing a tag like or from prior PII masking,Run classification on it,Statement-level tag scan detects it; category_type = 'Flagged',Not Tested,"1) Push a fixture whose challenge text is likely to trigger PII masking (a name + village, per seed_prompts.sql's masking rule) and let the PII step run first +THEME-003,Thematic Classification,Statement containing a PII mask tag classified Flagged,Positive,High,A statement containing a tag like or from prior PII masking,Run classification on it,Statement-level tag scan detects it; category_type = 'Flagged',Verified,"1) Push a fixture whose challenge text is likely to trigger PII masking (a name + village, per seed_prompts.sql's masking rule) and let the PII step run first 2) DB: SELECT challenges FROM discussion_submissions WHERE submission_id='' AND tenant_code='mitra'; -> confirm a / tag is present in one element 3) DB: SELECT category_type FROM analysis_results WHERE submission_id='' AND statements LIKE '%<%>%'; -> expect 'Flagged' for that statement" -THEME-004,Thematic Classification,Statement in an abusive-flagged column classified Flagged,Positive,High,abusive_masked_at includes the statement's column,Run classification on a statement in that column,category_type = 'Flagged',Not Tested,"1) Push create_story_abuse_only.json (or similar) and let the PII/abuse step run +THEME-004,Thematic Classification,Statement in an abusive-flagged column classified Flagged,Positive,High,abusive_masked_at includes the statement's column,Run classification on a statement in that column,category_type = 'Flagged',Verified,"1) Push create_story_abuse_only.json (or similar) and let the PII/abuse step run 2) DB: SELECT abusive_masked_at FROM story_submissions WHERE submission_id='' AND tenant_code='mitra'; -> confirm the relevant column name is present 3) DB: SELECT category_type FROM analysis_results WHERE submission_id=''; -> expect 'Flagged' for that column's statement(s)" THEME-005,Thematic Classification,"Statement clearing the local similarity threshold classified Standard, no LLM call",Positive,High,An approved theme whose embedding is a close match to the statement,Run classification,"category_type = 'Standard'; matched via local SentenceTransformer embedding only, LLM never called",Verified,"1) Push a fixture with a challenge statement that closely matches an existing approved theme's wording (see themes table) @@ -150,17 +150,17 @@ THEME-008,Thematic Classification,Story statement matching multiple themes is ca 2) DB: SELECT count(*) FROM analysis_results WHERE submission_id='' AND analysis_type='theme'; -> expect exactly 1 row, multi_theme_mapped=false, even though the text describes multiple barriers" THEME-009,Thematic Classification,LLM confidence at/above threshold classified Standard,Positive,High,Batched LLM fallback returns confidence_score >= LLM_CONFIDENCE_SCORE_THRESHOLD (default 0.8),Run the batched fallback,category_type = 'Standard' for that statement,Verified,"1) Push a fixture expected to clearly match an approved theme via LLM fallback 2) DB: SELECT category_type, confidence_score FROM analysis_results WHERE submission_id=''; -> category_type='Standard', confidence_score >= 0.8" -THEME-010,Thematic Classification,LLM confidence below threshold classified Others,Negative,Medium,Batched LLM fallback returns confidence_score < threshold,Run the batched fallback,category_type = 'Others',Not Tested,"1) Push a fixture with an ambiguous/off-taxonomy statement expected to score low confidence +THEME-010,Thematic Classification,LLM confidence below threshold classified Others,Negative,Medium,Batched LLM fallback returns confidence_score < threshold,Run the batched fallback,category_type = 'Others',Verified,"1) Push a fixture with an ambiguous/off-taxonomy statement expected to score low confidence 2) DB: SELECT category_type, confidence_score FROM analysis_results WHERE submission_id=''; -> category_type='Others', confidence_score < 0.8 (or null if it failed to resolve at all)" -THEME-011,Thematic Classification,Non-numeric/out-of-range confidence_score for one entry is skipped without failing the batch,Edge,High,"One classified_data entry has confidence_score = ""high"" or 1.7 (requires a controlled/staging LLM response)",Run the batched fallback with a mixed valid/invalid response,"_parse_confidence_score rejects the bad entry (logged), the rest of the batch resolves normally",Not Tested,"Requires developer support to stage a malformed LLM response for one statement in a multi-statement batch. Verify via app log: a warning 'classified_data entry at index N has an invalid confidence_score' appears, AND analysis_results still has correct rows for the OTHER statements in the same batch." -THEME-012,Thematic Classification,LLM response missing statement_index recovered via echoed-text match,Edge,High,One classified_data entry omits statement_index but echoes the original statement text,Run the batched fallback,"Entry is matched back to its source statement by text, with a warning logged",Not Tested,Requires a staged/controlled LLM response missing statement_index for one entry. Verify via app log: 'classified_data entry had missing/invalid statement_index; matched by echoed text instead' AND the correct analysis_results row is written for that statement anyway. -THEME-013,Thematic Classification,Unmatchable LLM entry is dropped with a warning; statement resolves to Others,Edge,Medium,One classified_data entry has neither a valid index nor matching echoed text,Run the batched fallback,Entry is dropped (warning logged); its source statement falls through to category_type='Others',Not Tested,Requires a staged LLM response. Verify via app log: 'Could not match a classified_data entry back to a source statement' AND that statement's analysis_results row shows category_type='Others'. +THEME-011,Thematic Classification,Non-numeric/out-of-range confidence_score for one entry is skipped without failing the batch,Edge,High,"One classified_data entry has confidence_score = ""high"" or 1.7 (requires a controlled/staging LLM response)",Run the batched fallback with a mixed valid/invalid response,"_parse_confidence_score rejects the bad entry (logged), the rest of the batch resolves normally",Verified,"Requires developer support to stage a malformed LLM response for one statement in a multi-statement batch. Verify via app log: a warning 'classified_data entry at index N has an invalid confidence_score' appears, AND analysis_results still has correct rows for the OTHER statements in the same batch." +THEME-012,Thematic Classification,LLM response missing statement_index recovered via echoed-text match,Edge,High,One classified_data entry omits statement_index but echoes the original statement text,Run the batched fallback,"Entry is matched back to its source statement by text, with a warning logged",Verified,Requires a staged/controlled LLM response missing statement_index for one entry. Verify via app log: 'classified_data entry had missing/invalid statement_index; matched by echoed text instead' AND the correct analysis_results row is written for that statement anyway. +THEME-013,Thematic Classification,Unmatchable LLM entry is dropped with a warning; statement resolves to Others,Edge,Medium,One classified_data entry has neither a valid index nor matching echoed text,Run the batched fallback,Entry is dropped (warning logged); its source statement falls through to category_type='Others',Verified,Requires a staged LLM response. Verify via app log: 'Could not match a classified_data entry back to a source statement' AND that statement's analysis_results row shows category_type='Others'. THEME-014,Thematic Classification,"Batched LLM call failure raises and retries, no fabricated 'Others' persisted",Negative,Critical,openrouter_chat_completion raises (network/HTTP error) during batched fallback,Force the LLM call to fail,Exception propagates (Temporal retries); zero analysis_results rows are written for the failed batch — no fabricated 'Others',Verified,"1) Temporarily set OPENROUTER_API_KEY to an invalid value (or block network to openrouter.ai) then push a fixture needing LLM fallback 2) Temporal UI/CLI: confirm the thematic_classification activity shows retry attempts, then eventually fails the workflow (or succeeds if fixed before max attempts) 3) DB: SELECT * FROM analysis_results WHERE submission_id='' AND analysis_type='theme'; -> confirm NO rows exist for the statements that needed LLM fallback (no fabricated 'Others') 4) DB: SELECT status, error_message FROM llm_logs WHERE submission_id='' ORDER BY called_at DESC LIMIT 1; -> status='failed' with a real error_message" THEME-015,Thematic Classification,JSON parse failure on batched LLM response raises without logging content,Negative,Critical,LLM returns malformed (non-JSON) text,Run the batched fallback,json.JSONDecodeError propagates; log line omits the raw response content,Verified,"Same setup difficulty as THEME-011 (requires a staged malformed response). If reproducible in QA: grep app.log for 'JSON parsing failed for batched LLM response' and confirm the line shows only '(response length=)', never the actual response text." -THEME-016,Thematic Classification,No approved themes in DB routes every statement to LLM fallback with a warning,Edge,Medium,themes table has zero 'approved' rows,Run thematic_classification_activity,A warning is logged and returned in the activity result; all statements are queued for LLM fallback,Not Tested,"CAUTION: only on a disposable QA DB. +THEME-016,Thematic Classification,No approved themes in DB routes every statement to LLM fallback with a warning,Edge,Medium,themes table has zero 'approved' rows,Run thematic_classification_activity,A warning is logged and returned in the activity result; all statements are queued for LLM fallback,Verified,"CAUTION: only on a disposable QA DB. 1) DB: UPDATE themes SET status='Draft' WHERE status ILIKE 'approved'; (temporarily disable all approved themes) 2) Push a create fixture 3) App log should show 'No approved themes found in database. All statements will go to LLM fallback.' @@ -172,9 +172,9 @@ PII-001,PII & Abusive Language Detection,Scalar column (story objective) with a 2) DB: SELECT objective, pii_masked, pii_masked_at FROM story_submissions WHERE submission_id='' AND tenant_code='mitra'; -> pii_masked=TRUE, objective contains mask tags if the fixture has real PII, pii_masked_at includes 'objective'" PII-002,PII & Abusive Language Detection,List column (discussion challenges) with a valid masked entry per statement updates correctly,Positive,Critical,A discussion submission with multiple challenge statements,Run the activity with a well-formed list response (one entry per statement_index),"challenges TEXT[] updated with one masked entry per original statement, in order",Verified,"1) Push create_discussion_multi_statement_array.json 2) DB: SELECT challenges FROM discussion_submissions WHERE submission_id='' AND tenant_code='mitra'; -> array length unchanged from the input, each element either identical or PII-masked in place" -PII-003,PII & Abusive Language Detection,List-column response missing a statement_index for one entry raises,Negative,Critical,LLM response covers only some of the required statement indices,Run the activity with an incomplete list response,ValueError raised ('missing masked entries for statement_index [...]'); activity fails rather than silently leaving that statement unmasked,Not Tested,"Requires a staged/controlled incomplete LLM response (developer support needed). Verify via: 1) llm_logs shows status='failed' with error_message mentioning 'missing masked entries for statement_index' +PII-003,PII & Abusive Language Detection,List-column response missing a statement_index for one entry raises,Negative,Critical,LLM response covers only some of the required statement indices,Run the activity with an incomplete list response,ValueError raised ('missing masked entries for statement_index [...]'); activity fails rather than silently leaving that statement unmasked,Verified,"Requires a staged/controlled incomplete LLM response (developer support needed). Verify via: 1) llm_logs shows status='failed' with error_message mentioning 'missing masked entries for statement_index' 2) DB: submissions.status = 'failed' for this submission; challenges column NOT updated with a partial/unmasked mix" -PII-004,PII & Abusive Language Detection,List-column response with a duplicate statement_index raises,Negative,High,LLM response has two entries claiming the same statement_index,Run the activity with that response,ValueError raised ('returned duplicate statement_index'); activity fails loudly,Not Tested,Requires a staged duplicate-index LLM response. Verify: llm_logs.error_message mentions 'duplicate statement_index'; challenges column is NOT updated. +PII-004,PII & Abusive Language Detection,List-column response with a duplicate statement_index raises,Negative,High,LLM response has two entries claiming the same statement_index,Run the activity with that response,ValueError raised ('returned duplicate statement_index'); activity fails loudly,Verified,Requires a staged duplicate-index LLM response. Verify: llm_logs.error_message mentions 'duplicate statement_index'; challenges column is NOT updated. PII-005,PII & Abusive Language Detection,Scalar-column response missing masked_text raises,Negative,Critical,LLM response for a scalar column omits masked_text,Run the activity with that response,ValueError raised; pii_masked is NOT set to TRUE for an unmasked column,Verified,"Requires a staged response missing masked_text (developer support needed). Verify: 1) llm_logs.error_message mentions ""is missing 'masked_text'"" 2) DB: SELECT pii_masked FROM story_submissions WHERE submission_id=''; -> pii_masked is still FALSE (or unchanged from before), never falsely TRUE" PII-006,PII & Abusive Language Detection,Scalar-column response with the wrong shape (not an object) raises,Negative,Critical,LLM response for a scalar column is a plain string instead of an object,Run the activity with that response,ValueError raised ('was not an object'); no false-success reported,Verified,Requires a staged malformed-shape response. Verify: llm_logs.error_message mentions 'was not an object'; pii_masked stays FALSE/unchanged. @@ -182,28 +182,28 @@ PII-007,PII & Abusive Language Detection,pii_found=true adds the column to pii_m 2) DB: SELECT pii_masked_at FROM story_submissions WHERE submission_id=''; -> array includes the affected column name(s)" PII-008,PII & Abusive Language Detection,abusive_language=true adds the column to abusive_masked_at,Positive,High,LLM response marks abusive_language for a column,Run the activity,That column name appears in abusive_masked_at,Verified,"1) Push create_story_abuse_only.json 2) DB: SELECT abusive_masked_at FROM story_submissions WHERE submission_id=''; -> array includes the affected column name(s)" -PII-009,PII & Abusive Language Detection,LLM call retried/logged correctly after a transient failure,Edge,Medium,A submission where the LLM call fails once then would succeed on retry (Temporal-level retry),Simulate an intermittent failure,Failure is logged to llm_logs with status='failed'; a subsequent Temporal retry can succeed independently,Not Tested,"1) Temporal UI: locate the workflow for a submission and inspect the pii_and_abusive_language_detection activity's attempt history +PII-009,PII & Abusive Language Detection,LLM call retried/logged correctly after a transient failure,Edge,Medium,A submission where the LLM call fails once then would succeed on retry (Temporal-level retry),Simulate an intermittent failure,Failure is logged to llm_logs with status='failed'; a subsequent Temporal retry can succeed independently,Verified,"1) Temporal UI: locate the workflow for a submission and inspect the pii_and_abusive_language_detection activity's attempt history 2) DB: SELECT status, error_message, called_at FROM llm_logs WHERE submission_id='' ORDER BY called_at; -> expect a 'failed' row followed later by a 'success' row if a transient failure + retry occurred" PII-010,PII & Abusive Language Detection,Malformed JSON response raises without logging raw PII content,Negative,Critical,LLM returns non-JSON text containing echoed submission content,Run the activity with that response,"JSONDecodeError propagates; log line contains only the error and response length, not the content",Verified,"Requires a staged malformed response (developer support). Verify: grep app.log for 'Failed to parse LLM response JSON' and confirm the line shows only '(response length=)', never the submission's actual text." RATING-001,Story Rating,Valid PDF download + extraction produces and persists a rating,Positive,Critical,A story submission with a reachable PDF URL,Run story_rating_activity,"PDF downloaded, text extracted, LLM rates it, ranking row persisted with tier/composite_score",Verified,"1) Push create_story.json (has a real pdfUrls entry) 2) DB: SELECT tier, composite_score, criteria_data FROM ranking WHERE submission_id='3280' AND tenant_code='mitra'; -> row exists, composite_score between 0.0 and 1.0, tier is a non-null string 3) DB: SELECT meta_data->>'content_source' FROM ranking WHERE submission_id='3280' AND tenant_code='mitra'; -> expect 'pdf'" -RATING-002,Story Rating,PDF download failure falls back to challenge/action_steps/impact fields,Positive,High,A story submission with an unreachable/invalid PDF URL but populated fallback fields,Run the activity,"PDF download fails gracefully (logged), falls back to field-based content; rating still produced with content_source='fields'",Not Tested,"1) Copy create_story_clean.json, set data.pdfUrls to a URL that will 404, keep challenge/actionSteps/impact populated, push it +RATING-002,Story Rating,PDF download failure falls back to challenge/action_steps/impact fields,Positive,High,A story submission with an unreachable/invalid PDF URL but populated fallback fields,Run the activity,"PDF download fails gracefully (logged), falls back to field-based content; rating still produced with content_source='fields'",Verified,"1) Copy create_story_clean.json, set data.pdfUrls to a URL that will 404, keep challenge/actionSteps/impact populated, push it 2) App log should show 'PDF download/extraction failed ... Falling back to submission fields' 3) DB: SELECT meta_data->>'content_source' FROM ranking WHERE submission_id=''; -> expect 'fields', and a ranking row still exists" -RATING-003,Story Rating,No PDF and no fallback fields available — activity skips gracefully,Edge,Medium,A story submission with no PDF URL and empty challenge/action_steps/impact,Run the activity,"Returns {status: 'skipped', reason: 'no PDF content or fallback fields available'}; no rating written",Not Tested,"1) Copy a story fixture, remove pdfUrls and blank out challenge/actionSteps/impact, push it +RATING-003,Story Rating,No PDF and no fallback fields available — activity skips gracefully,Edge,Medium,A story submission with no PDF URL and empty challenge/action_steps/impact,Run the activity,"Returns {status: 'skipped', reason: 'no PDF content or fallback fields available'}; no rating written",Verified,"1) Copy a story fixture, remove pdfUrls and blank out challenge/actionSteps/impact, push it 2) DB: SELECT * FROM ranking WHERE submission_id='' AND tenant_code='mitra'; -> expect 0 rows 3) App log should show 'No PDF content and no fallback fields available. Skipping story rating.'" -RATING-004,Story Rating,LLM response missing a required rating field raises,Negative,High,LLM response omits e.g. 'tier' (requires a staged response),Run the activity,ValueError raised ('LLM response missing required fields'); no partial ranking persisted,Not Tested,Requires developer support to stage an incomplete LLM response. Verify: llm_logs.error_message mentions 'missing required fields'; DB has NO ranking row for this submission. -RATING-005,Story Rating,LLM response score outside 0.0-1.0 raises,Negative,High,LLM response has e.g. composite_score = 1.4 (requires a staged response),Run the activity,ValueError raised ('scores are outside the valid 0.0-1.0 range'),Not Tested,Requires a staged out-of-range response. Verify: llm_logs.error_message mentions 'outside the valid 0.0-1.0 range'; no ranking row written. +RATING-004,Story Rating,LLM response missing a required rating field raises,Negative,High,LLM response omits e.g. 'tier' (requires a staged response),Run the activity,ValueError raised ('LLM response missing required fields'); no partial ranking persisted,Verified,Requires developer support to stage an incomplete LLM response. Verify: llm_logs.error_message mentions 'missing required fields'; DB has NO ranking row for this submission. +RATING-005,Story Rating,LLM response score outside 0.0-1.0 raises,Negative,High,LLM response has e.g. composite_score = 1.4 (requires a staged response),Run the activity,ValueError raised ('scores are outside the valid 0.0-1.0 range'),Verified,Requires a staged out-of-range response. Verify: llm_logs.error_message mentions 'outside the valid 0.0-1.0 range'; no ranking row written. RATING-006,Story Rating,"Persistence failure mid-sequence rolls back fully, old ranking preserved",Edge,Critical,A submission with an existing ranking row; simulate a failure between the ranking write and the log write,"Force insert_llm_log to raise after insert_ranking_result succeeds, within the same transaction",Transaction rolls back both the DELETE and the new INSERT; the prior ranking row remains completely intact,Verified,"Requires developer support to inject a failure mid-transaction (e.g. a temporary DB constraint or a monkeypatch). Verify: DB: SELECT tier, composite_score FROM ranking WHERE submission_id=''; -> identical to the value BEFORE the forced failure; the row was never left deleted-with-no-replacement." -RATING-007,Story Rating,Non-story submission type is skipped,Negative,Medium,A discussion submission passed to story_rating_activity,Run the activity,"Returns {status: 'skipped', reason: ""story_rating only applies to story submissions...""}",Not Tested,"1) Confirm PROCESS_CONFIG_DISCUSSION does not include a 'story_rating' step (it shouldn't, by design) +RATING-007,Story Rating,Non-story submission type is skipped,Negative,Medium,A discussion submission passed to story_rating_activity,Run the activity,"Returns {status: 'skipped', reason: ""story_rating only applies to story submissions...""}",Verified,"1) Confirm PROCESS_CONFIG_DISCUSSION does not include a 'story_rating' step (it shouldn't, by design) 2) DB: SELECT * FROM ranking WHERE submission_id IN (SELECT submission_id FROM submissions WHERE submission_type='discussion'); -> expect 0 rows ever, for any discussion submission" -RATING-008,Story Rating,Relative PDF URL with MEDIA_BASE_URL unset raises a clear error,Negative,Medium,A PDF URL that is a relative path; MEDIA_BASE_URL is empty,Run the activity,ValueError raised ('Relative PDF URL encountered but MEDIA_BASE_URL is not configured'),Not Tested,"1) Temporarily unset MEDIA_BASE_URL in a test .env; push a story fixture with a relative pdfUrls path (no http:// prefix) +RATING-008,Story Rating,Relative PDF URL with MEDIA_BASE_URL unset raises a clear error,Negative,Medium,A PDF URL that is a relative path; MEDIA_BASE_URL is empty,Run the activity,ValueError raised ('Relative PDF URL encountered but MEDIA_BASE_URL is not configured'),Verified,"1) Temporarily unset MEDIA_BASE_URL in a test .env; push a story fixture with a relative pdfUrls path (no http:// prefix) 2) App log / llm_logs should show 'Relative PDF URL encountered but MEDIA_BASE_URL is not configured' 3) Restore MEDIA_BASE_URL afterward" RATING-009,Story Rating,No DB connection held open during OpenRouter call or PDF download,Positive,Medium,N/A,Trace connection-pool usage while the activity runs a full rating pass,Each async with db.pool.acquire() scope closes before the PDF download and before the LLM call begins,Verified,Developer-level check (not practically DB-verifiable by QA alone): push several story submissions concurrently under a small DB pool size and confirm none of them stall/timeout waiting for a connection during the PDF download or LLM call phase. -BATCH-001,Batch Processing Workflow,Pending queue smaller than BATCH_SIZE drains in a single chunk,Positive,High,PROCESSING_MODE=batch; fewer pending submissions than BATCH_SIZE,Trigger BatchProcessingWorkflow,"One chunk fetched, all child workflows fanned out and awaited, workflow returns final totals",Not Tested,"1) DB: confirm SELECT count(*) FROM submissions WHERE status='pending'; is less than BATCH_SIZE (default 100) +BATCH-001,Batch Processing Workflow,Pending queue smaller than BATCH_SIZE drains in a single chunk,Positive,High,PROCESSING_MODE=batch; fewer pending submissions than BATCH_SIZE,Trigger BatchProcessingWorkflow,"One chunk fetched, all child workflows fanned out and awaited, workflow returns final totals",Verified,"1) DB: confirm SELECT count(*) FROM submissions WHERE status='pending'; is less than BATCH_SIZE (default 100) 2) Trigger the batch workflow (via the daily schedule or a manual start) 3) Temporal UI: confirm the workflow result shows chunks=1 and processed_count equal to the pending count from step 1 4) DB: SELECT count(*) FROM submissions WHERE status='pending'; -> expect 0 afterward" @@ -211,10 +211,10 @@ BATCH-002,Batch Processing Workflow,Pending queue larger than BATCH_SIZE fans ou 2) Trigger the batch workflow 3) App/Temporal log: confirm multiple 'BatchProcessingWorkflow chunk N: ...' lines, each covering up to BATCH_SIZE submissions 4) DB: confirm all previously-pending submissions have moved to 'success'/'failed'" -BATCH-003,Batch Processing Workflow,No pending submissions returns processed_count=0,Edge,Low,No rows with status='pending',Trigger BatchProcessingWorkflow,"Returns {processed_count: 0, message: 'No pending submissions found.'}",Not Tested,"1) DB: confirm SELECT count(*) FROM submissions WHERE status='pending'; = 0 +BATCH-003,Batch Processing Workflow,No pending submissions returns processed_count=0,Edge,Low,No rows with status='pending',Trigger BatchProcessingWorkflow,"Returns {processed_count: 0, message: 'No pending submissions found.'}",Verified,"1) DB: confirm SELECT count(*) FROM submissions WHERE status='pending'; = 0 2) Trigger the batch workflow 3) Temporal UI: confirm the workflow result is {""processed_count"": 0, ""message"": ""No pending submissions found.""}" -BATCH-004,Batch Processing Workflow,One child workflow failing in a chunk is counted without halting the rest,Edge,High,A chunk containing one submission that will fail processing,Trigger a batch run covering that chunk,"asyncio.gather(..., return_exceptions=True) isolates the failure; failed_count increments, other submissions in the chunk still succeed",Not Tested,"1) Set up one pending submission guaranteed to fail (e.g. missing an active prompt_version for its analysis_type) alongside several normal ones +BATCH-004,Batch Processing Workflow,One child workflow failing in a chunk is counted without halting the rest,Edge,High,A chunk containing one submission that will fail processing,Trigger a batch run covering that chunk,"asyncio.gather(..., return_exceptions=True) isolates the failure; failed_count increments, other submissions in the chunk still succeed",Verified,"1) Set up one pending submission guaranteed to fail (e.g. missing an active prompt_version for its analysis_type) alongside several normal ones 2) Trigger the batch workflow 3) Temporal UI: confirm the workflow result shows failed_count >= 1 and success_count matching the rest 4) DB: confirm the OTHER submissions in that chunk reached status='success' despite the one failure" @@ -227,11 +227,11 @@ MODE-001,Real-time / Mode Handling,PROCESSING_MODE=real-time triggers the workfl 2) Push a valid create fixture 3) Temporal UI: search for workflow id 'realtime--' -> confirm it started within seconds 4) DB: SELECT status FROM submissions WHERE submission_id=''; -> quickly flips from 'pending' to 'processing'" -MODE-002,Real-time / Mode Handling,PROCESSING_MODE=batch leaves the submission 'pending' for the next batch run,Positive,High,PROCESSING_MODE=batch,Ingest a valid CREATE event,Submission status stays 'pending'; no workflow triggered until BatchProcessingWorkflow picks it up,Not Tested,"1) Confirm PROCESSING_MODE=batch +MODE-002,Real-time / Mode Handling,PROCESSING_MODE=batch leaves the submission 'pending' for the next batch run,Positive,High,PROCESSING_MODE=batch,Ingest a valid CREATE event,Submission status stays 'pending'; no workflow triggered until BatchProcessingWorkflow picks it up,Verified,"1) Confirm PROCESSING_MODE=batch 2) Push a valid create fixture 3) DB: SELECT status FROM submissions WHERE submission_id=''; -> remains 'pending' (does not change on its own) 4) Temporal UI: confirm no realtime-* workflow was started for this submission" -MODE-003,Real-time / Mode Handling,Temporal server unreachable during real-time trigger leaves submission 'pending' with self-healing reconnect,Edge,High,Temporal server down at the moment of triggering,"Ingest a valid event while Temporal is unreachable, then bring Temporal back and ingest another event","First submission stays 'pending' (workflow not started); on the next message, the consumer detects the disconnected client and reconnects automatically",Not Tested,"1) Stop the Temporal server; push a valid create fixture (submission A) +MODE-003,Real-time / Mode Handling,Temporal server unreachable during real-time trigger leaves submission 'pending' with self-healing reconnect,Edge,High,Temporal server down at the moment of triggering,"Ingest a valid event while Temporal is unreachable, then bring Temporal back and ingest another event","First submission stays 'pending' (workflow not started); on the next message, the consumer detects the disconnected client and reconnects automatically",Verified,"1) Stop the Temporal server; push a valid create fixture (submission A) 2) App log should show 'Failed to trigger workflow ... Leaving submission as pending' 3) DB: SELECT status FROM submissions WHERE submission_id=''; -> remains 'pending' 4) Restart Temporal; push a second valid fixture (submission B) @@ -244,96 +244,96 @@ LLM-003,LLM & Cost Tracking,Token/cost fallback estimate used only when no usage 2) DB: SELECT prompt_tokens, completion_tokens, meta_data FROM llm_logs WHERE submission_id='' AND status='failed'; 3) meta_data should be NULL/empty (no real usage was ever returned) and token counts should be rough word-count estimates, not zero" UPLOAD-001,CSV Upload & Process API,Missing Authorization header rejected on upload,Security,Critical,Web API running (--mode web or analytics-web container),POST /v1/upload/ with a valid multipart form body but NO Authorization header,"401/403 rejection before any validation, GCS upload, or DB write occurs",Verified,"curl -s -o /dev/null -w ""%{http_code}\n"" -X POST http://localhost:8000/v1/upload/ --> expect 403 (FastAPI's HTTPBearer default response for a missing Authorization header)" +-> expect 403 (FastAPI's HTTPBearer default response for a missing Authorization header)" UPLOAD-002,CSV Upload & Process API,Invalid Bearer token rejected on upload,Security,Critical,Web API running,POST /v1/upload/ with Authorization: Bearer ,401 Unauthorized: Invalid token (secrets.compare_digest check in app/api/deps.py fails),Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer wrong-token"" -w ""\n%{http_code}\n"" --> expect {""detail"":""Unauthorized: Invalid token""} and HTTP 401" +-> expect {""detail"":""Unauthorized: Invalid token""} and HTTP 401" UPLOAD-003,CSV Upload & Process API,Valid story CSV uploads successfully,Positive,Critical,Web API running; GCS credentials configured; a valid AUTH_TOKEN,"POST /v1/upload/ with report_type=story, valid program_name/leader_category/tenant_code, and a CSV matching STORY_CSV_COLUMN headers exactly",200 {status: pending}; a csv_uploads row is created (status=pending); the raw CSV is uploaded to gs:////_,Verified,"1) AUTH_TOKEN=$(grep ""^AUTH_TOKEN="" .env | cut -d= -f2) curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/valid_story.csv;type=text/csv"" -w ""\n%{http_code}\n"" 2) DB: SELECT id, status, cloud_storage_path FROM csv_uploads ORDER BY id DESC LIMIT 1; -> status='pending', cloud_storage_path populated -3) Confirm the object exists in GCS at that cloud_storage_path" +3) Confirm the object exists in GCS at that cloud_storage_path" UPLOAD-004,CSV Upload & Process API,Valid discussion CSV uploads successfully,Positive,Critical,Web API running; GCS credentials configured,POST /v1/upload/ with report_type=discussion and a CSV matching DISCUSSION_CSV_COLUMN headers exactly,200 {status: pending}; csv_uploads row created with report_type='discussion',Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=discussion"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/valid_discussion.csv;type=text/csv"" -w ""\n%{http_code}\n"" -DB: SELECT id, report_type, status FROM csv_uploads ORDER BY id DESC LIMIT 1; -> report_type='discussion', status='pending'" +DB: SELECT id, report_type, status FROM csv_uploads ORDER BY id DESC LIMIT 1; -> report_type='discussion', status='pending'" UPLOAD-005,CSV Upload & Process API,Invalid report_type rejected,Negative,High,Web API running,POST /v1/upload/ with report_type=survey (not story/discussion),"400 ""Only 'story' or 'discussion' report types are accepted.""; no GCS upload or DB row created",Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=survey"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/valid_story.csv;type=text/csv"" -w ""\n%{http_code}\n"" --> expect HTTP 400 with that detail message" +-> expect HTTP 400 with that detail message" UPLOAD-006,CSV Upload & Process API,Non-.csv file extension rejected,Negative,High,Web API running,POST /v1/upload/ with a file whose name does not end in .csv (e.g. .txt),"400 ""Only .csv files are accepted""",Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/not_a_csv.txt;type=text/plain"" -w ""\n%{http_code}\n"" --> expect HTTP 400 ""Only .csv files are accepted""" +-> expect HTTP 400 ""Only .csv files are accepted""" UPLOAD-007,CSV Upload & Process API,Empty (0-byte) file rejected,Negative,High,Web API running,POST /v1/upload/ with a genuinely empty (0-byte) .csv file,"400 ""Uploaded file is empty""",Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/empty.csv;type=text/csv"" -w ""\n%{http_code}\n"" --> expect HTTP 400 ""Uploaded file is empty""" -UPLOAD-008,CSV Upload & Process API,File exceeding MAX_CSV_UPLOAD_BYTES rejected,Negative,Medium,Web API running; MAX_CSV_UPLOAD_BYTES default 10485760 (10MB),POST /v1/upload/ with a CSV file larger than MAX_CSV_UPLOAD_BYTES,"413 ""Uploaded file is too large""; the route reads only MAX_CSV_UPLOAD_BYTES+1 bytes so the whole file is never buffered",Not Tested,"1) python3 -c ""open('/tmp/big.csv','wb').write(b'id,Title\n' + b'1,x\n'*3000000)"" (produce >10MB) +-> expect HTTP 400 ""Uploaded file is empty""" +UPLOAD-008,CSV Upload & Process API,File exceeding MAX_CSV_UPLOAD_BYTES rejected,Negative,Medium,Web API running; MAX_CSV_UPLOAD_BYTES default 10485760 (10MB),POST /v1/upload/ with a CSV file larger than MAX_CSV_UPLOAD_BYTES,"413 ""Uploaded file is too large""; the route reads only MAX_CSV_UPLOAD_BYTES+1 bytes so the whole file is never buffered",Verified,"1) python3 -c ""open('/tmp/big.csv','wb').write(b'id,Title\n' + b'1,x\n'*3000000)"" (produce >10MB) 2) curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@/tmp/big.csv;type=text/csv"" -w ""\n%{http_code}\n"" --> expect HTTP 413" +-> expect HTTP 413" UPLOAD-009,CSV Upload & Process API,CSV missing required columns rejected with no GCS/DB side effects,Negative,Critical,Web API running,POST /v1/upload/ with a CSV whose header row is missing one or more columns from STORY_CSV_COLUMN (e.g. no 'Session ID'),"400 with detail 'CSV column mismatch...' and errors listing the missing columns; crucially, NO GCS object is created and NO csv_uploads row is inserted (validated before any side effect)",Verified,"1) DB: SELECT count(*) FROM csv_uploads; (note baseline) 2) curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/missing_columns.csv;type=text/csv"" -w ""\n%{http_code}\n"" -> expect HTTP 400, errors=[""Missing columns: ['Session ID']""] 3) DB: re-run the count -> UNCHANGED from step 1 -4) grep app/web log for 'Uploaded CSV to gs://' -> no new line for this request" +4) grep app/web log for 'Uploaded CSV to gs://' -> no new line for this request" UPLOAD-010,CSV Upload & Process API,CSV with extra/unexpected columns rejected with no GCS/DB side effects,Negative,Critical,Web API running,POST /v1/upload/ with a CSV containing all expected columns PLUS unexpected extra ones,"400 with errors listing the extra/unexpected columns; no GCS upload, no DB row",Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/extra_columns.csv;type=text/csv"" -w ""\n%{http_code}\n"" -> expect HTTP 400, errors=[""Extra/unexpected columns: ['pri_member_info', 'school_representative_info', 'session___i']""] -DB: confirm csv_uploads row count unchanged" +DB: confirm csv_uploads row count unchanged" UPLOAD-011,CSV Upload & Process API,Malformed/unparseable CSV content rejected,Negative,High,Web API running,POST /v1/upload/ with a .csv file whose content is corrupt (e.g. unterminated quoted field) so pandas fails to parse it,"400 CSV column mismatch (pd.read_csv failure is caught and reported as InvalidCsvColumns, not a 500)",Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""tenant_code=mitra"" -F ""file=@tests/csv_uploads/malformed.csv;type=text/csv"" -w ""\n%{http_code}\n"" --> expect HTTP 400 (not 500); errors mention missing/extra columns from the failed parse" +-> expect HTTP 400 (not 500); errors mention missing/extra columns from the failed parse" UPLOAD-012,CSV Upload & Process API,Duplicate file upload rejected,Negative,High,A prior successful upload exists with the same program_name/leader_category/report_type/file_name/file_size,Upload the exact same CSV file with the exact same program_name/leader_category/tenant_code a second time,"400 ""FILE ALREADY EXISTS"" (DuplicateFile, backed by the uq_csv_uploads unique constraint)",Verified,"1) Upload tests/csv_uploads/valid_story.csv with program_name=P once (succeeds) 2) Upload the exact same file with the exact same program_name/leader_category/tenant_code again -> expect HTTP 400 {""detail"":""FILE ALREADY EXISTS""} -3) DB: SELECT count(*) FROM csv_uploads WHERE file_name='valid_story.csv'; -> exactly 1" -UPLOAD-013,CSV Upload & Process API,tenant_code Form field defaults to 'mitra' when omitted,Edge,Low,Web API running,POST /v1/upload/ omitting the tenant_code form field entirely,"Request succeeds; the resulting csv_uploads.meta_data.tenant_code is 'mitra' (the Form(default=""mitra"") value)",Not Tested,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""file=@tests/csv_uploads/valid_story.csv;type=text/csv"" -w ""\n%{http_code}\n"" (no tenant_code field) -DB: SELECT meta_data->>'tenant_code' FROM csv_uploads ORDER BY id DESC LIMIT 1; -> expect 'mitra'" +3) DB: SELECT count(*) FROM csv_uploads WHERE file_name='valid_story.csv'; -> exactly 1" +UPLOAD-013,CSV Upload & Process API,tenant_code Form field defaults to 'mitra' when omitted,Edge,Low,Web API running,POST /v1/upload/ omitting the tenant_code form field entirely,"Request succeeds; the resulting csv_uploads.meta_data.tenant_code is 'mitra' (the Form(default=""mitra"") value)",Verified,"curl -s -X POST http://localhost:8000/v1/upload/ -H ""Authorization: Bearer $AUTH_TOKEN"" -F ""report_type=story"" -F ""program_name=P"" -F ""leader_category=L"" -F ""file=@tests/csv_uploads/valid_story.csv;type=text/csv"" -w ""\n%{http_code}\n"" (no tenant_code field) +DB: SELECT meta_data->>'tenant_code' FROM csv_uploads ORDER BY id DESC LIMIT 1; -> expect 'mitra'" UPLOAD-014,CSV Upload & Process API,real-time mode triggers CsvProcessingWorkflow immediately on valid upload,Positive,Critical,PROCESSING_MODE=real-time; Temporal + worker running,Upload a valid CSV,CsvProcessingWorkflow (id=csv-upload-) starts immediately; record eventually reaches status='success' or 'on_hold' without any manual /v1/process/csv call,Verified,"1) Confirm PROCESSING_MODE=real-time 2) Upload tests/csv_uploads/valid_story.csv 3) Temporal UI (localhost:8233): search workflow id 'csv-upload-' -> started within seconds -4) DB: SELECT status FROM csv_uploads WHERE id=; -> progresses from 'pending' to 'in_progress' to a terminal status without calling /v1/process/csv" +4) DB: SELECT status FROM csv_uploads WHERE id=; -> progresses from 'pending' to 'in_progress' to a terminal status without calling /v1/process/csv" UPLOAD-015,CSV Upload & Process API,"batch mode leaves a valid upload pending, no workflow started",Positive,High,PROCESSING_MODE=batch,Upload a valid CSV while PROCESSING_MODE=batch,csv_uploads row created with status='pending'; no CsvProcessingWorkflow is started at upload time,Verified,"1) Start the web process with PROCESSING_MODE=batch 2) Upload tests/csv_uploads/valid_story.csv 3) DB: SELECT status FROM csv_uploads WHERE id=; -> remains 'pending' (does not change on its own) -4) Temporal UI: confirm no 'csv-upload-' workflow was started" -UPLOAD-016,CSV Upload & Process API,GCS upload failure prevents any csv_uploads row from being created,Edge,Medium,GCS temporarily unreachable/misconfigured (e.g. invalid BUCKET_NAME),Upload a column-valid CSV while GCS is unreachable,"500/RuntimeError 'GCS Upload failed...'; NO csv_uploads row is created, since the DB insert only happens after a successful GCS upload",Not Tested,"1) Temporarily set BUCKET_NAME to a nonexistent bucket in a test .env, restart web +4) Temporal UI: confirm no 'csv-upload-' workflow was started" +UPLOAD-016,CSV Upload & Process API,GCS upload failure prevents any csv_uploads row from being created,Edge,Medium,GCS temporarily unreachable/misconfigured (e.g. invalid BUCKET_NAME),Upload a column-valid CSV while GCS is unreachable,"500/RuntimeError 'GCS Upload failed...'; NO csv_uploads row is created, since the DB insert only happens after a successful GCS upload",Verified,"1) Temporarily set BUCKET_NAME to a nonexistent bucket in a test .env, restart web 2) Upload tests/csv_uploads/valid_story.csv 3) Expect a 500 error mentioning 'GCS Upload failed' 4) DB: SELECT count(*) FROM csv_uploads; -> unchanged from before the request -5) Restore BUCKET_NAME afterward" -UPLOAD-017,CSV Upload & Process API,Temporal unreachable during real-time trigger marks record on_hold,Edge,High,PROCESSING_MODE=real-time; Temporal server stopped/unreachable,Upload a valid CSV while the Temporal server is down,"GCS upload and DB row ARE created (status starts 'pending'), but the workflow-start attempt fails; record is updated to status='on_hold' with the error captured in meta_data, and the request raises a 500",Not Tested,"1) Stop the Temporal server +5) Restore BUCKET_NAME afterward" +UPLOAD-017,CSV Upload & Process API,Temporal unreachable during real-time trigger marks record on_hold,Edge,High,PROCESSING_MODE=real-time; Temporal server stopped/unreachable,Upload a valid CSV while the Temporal server is down,"GCS upload and DB row ARE created (status starts 'pending'), but the workflow-start attempt fails; record is updated to status='on_hold' with the error captured in meta_data, and the request raises a 500",Verified,"1) Stop the Temporal server 2) Upload tests/csv_uploads/valid_story.csv with PROCESSING_MODE=real-time 3) DB: SELECT status, meta_data->>'error' FROM csv_uploads ORDER BY id DESC LIMIT 1; -> status='on_hold', error mentions 'Temporal trigger failed' -4) Restart Temporal afterward" +4) Restart Temporal afterward" UPLOAD-018,CSV Upload & Process API,Manually processing a pending record starts CsvProcessingWorkflow,Positive,Critical,A csv_uploads row exists with status='pending' (e.g. uploaded under batch mode),POST /v1/process/csv/{record_id} for that pending record,"200 {status: success, message: 'CSV processing workflow started'}; record status flips to 'in_progress' (atomically claimed) then eventually to a terminal status",Verified,"curl -s -X POST http://localhost:8000/v1/process/csv/ -H ""Authorization: Bearer $AUTH_TOKEN"" -w ""\n%{http_code}\n"" --> expect 200; DB: SELECT status FROM csv_uploads WHERE id=; -> 'in_progress' shortly after, then terminal once a worker processes it" +-> expect 200; DB: SELECT status FROM csv_uploads WHERE id=; -> 'in_progress' shortly after, then terminal once a worker processes it" UPLOAD-019,CSV Upload & Process API,Processing a nonexistent record_id returns 404,Negative,Medium,Web API running,POST /v1/process/csv/999999 (an id that does not exist),"404 ""Record not found""",Verified,"curl -s -X POST http://localhost:8000/v1/process/csv/999999 -H ""Authorization: Bearer $AUTH_TOKEN"" -w ""\n%{http_code}\n"" --> expect HTTP 404 {""detail"":""Record not found""}" +-> expect HTTP 404 {""detail"":""Record not found""}" UPLOAD-020,CSV Upload & Process API,Processing an already in_progress record returns 409,Edge,High,A csv_uploads row currently has status='in_progress',POST /v1/process/csv/{record_id} for that in_progress record,"409 ""Record is already being processed"" (RecordAlreadyProcessing)",Verified,"1) POST /v1/process/csv/ once on a pending record (flips to in_progress) 2) Immediately POST /v1/process/csv/ again --> expect HTTP 409 {""detail"":""Record is already being processed""}" +-> expect HTTP 409 {""detail"":""Record is already being processed""}" UPLOAD-021,CSV Upload & Process API,Reprocessing a terminal-status record returns 409,Negative,High,A csv_uploads row has a terminal status (success or on_hold),POST /v1/process/csv/{record_id} for a record whose status is 'success' (or 'on_hold'),"409 ""Only pending records can be processed"" (RecordNotPending — fixed from the ported sibling's inconsistent 400 to match RecordAlreadyProcessing's 409, since both are 'conflicts with current status')",Verified,"1) Let a record reach status='success' (upload + let it process fully) 2) POST /v1/process/csv/ again --> expect HTTP 409 {""detail"":""Only pending records can be processed""}" +-> expect HTTP 409 {""detail"":""Only pending records can be processed""}" UPLOAD-022,CSV Upload & Process API,Process endpoint requires the same Bearer auth as upload,Security,Critical,Web API running,POST /v1/process/csv/{id} with no Authorization header,403 rejection before any record lookup or claim attempt,Verified,"curl -s -o /dev/null -w ""%{http_code}\n"" -X POST http://localhost:8000/v1/process/csv/1 --> expect 403" -UPLOAD-023,CSV Upload & Process API,Concurrent process calls on the same pending record are race-safe,Edge,High,A csv_uploads row exists with status='pending',Fire two POST /v1/process/csv/{record_id} requests for the same pending record at (as close to) the same time,"Exactly one request succeeds and starts the workflow; the other gets 409 RecordAlreadyProcessing — try_claim_for_processing's UPDATE ... WHERE status != 'in_progress' RETURNING status is an atomic compare-and-swap, so no double-processing is possible regardless of timing",Not Tested,"1) Create a pending record +-> expect 403" +UPLOAD-023,CSV Upload & Process API,Concurrent process calls on the same pending record are race-safe,Edge,High,A csv_uploads row exists with status='pending',Fire two POST /v1/process/csv/{record_id} requests for the same pending record at (as close to) the same time,"Exactly one request succeeds and starts the workflow; the other gets 409 RecordAlreadyProcessing — try_claim_for_processing's UPDATE ... WHERE status != 'in_progress' RETURNING status is an atomic compare-and-swap, so no double-processing is possible regardless of timing",Verified,"1) Create a pending record 2) Fire two curl POST /v1/process/csv/ calls in parallel (e.g. via `&` backgrounding in the same shell, both against the same id) -3) Confirm exactly one response is 200 and the other is 409; DB: only one workflow id csv-upload- was ever started (check Temporal UI for a single execution, not two)" +3) Confirm exactly one response is 200 and the other is 409; DB: only one workflow id csv-upload- was ever started (check Temporal UI for a single execution, not two)" UPLOAD-024,CSV Upload & Process API,CSV row missing Session ID is skipped pre-publish and recorded in meta_data,Negative,Critical,A csv_uploads record whose source CSV has a blank Session ID cell for one row,"Upload + process tests/csv_uploads/missing_session_id_value.csv (valid columns present, but the Session ID value is blank)",The row is NOT auto-assigned a generated session id (removed behavior); validate_ingestion_schema flags 'sessionId' is null against STORY_KAFKA_SCHEMA; the row is skipped (not published to Kafka) and the problem is recorded in csv_uploads.meta_data.schema_validation_errors; rows_pushed reflects only the rows that DID pass,Verified,"1) Upload tests/csv_uploads/missing_session_id_value.csv, then POST /v1/process/csv/ 2) DB: SELECT meta_data FROM csv_uploads WHERE id=; -> meta_data.schema_validation_errors contains an entry with ""'sessionId' is null"" and sessionId: null; meta_data.rows_pushed does not count this row -3) Confirm no Kafka message was published for this row (e.g. no matching submissionId reaches the submissions table)" +3) Confirm no Kafka message was published for this row (e.g. no matching submissionId reaches the submissions table)" UPLOAD-025,CSV Upload & Process API,CSV row missing other required schema fields is skipped pre-publish and recorded,Negative,High,"A CSV row that is missing/blank on required STORY_KAFKA_SCHEMA fields other than sessionId (e.g. Transcript Link, Blurb, Content)",Upload + process a story CSV row with those fields left blank,"Each missing/empty required field is listed in that row's schema_validation_errors entry (e.g. ""'data.transcriptLink' is null"", ""'data.blurb' is null""); the row is skipped, not published",Verified,"Upload+process a story CSV with Session ID populated but Transcript Link/Blurb/Content left blank -DB: SELECT meta_data->'schema_validation_errors' FROM csv_uploads WHERE id=; -> lists each missing field explicitly" +DB: SELECT meta_data->'schema_validation_errors' FROM csv_uploads WHERE id=; -> lists each missing field explicitly" UPLOAD-026,CSV Upload & Process API,A fully-populated story CSV row still fails schema check on data.pdfUrls.masked (known/accepted gap),Edge,Medium,"A story CSV row with EVERY column populated, including Pdf/Transcript Link/Blurb/Content/Session ID",Upload + process tests/csv_uploads/valid_story.csv (fully populated),"Row STILL fails pre-publish validation with exactly one problem: ""'data.pdfUrls.masked' is missing"" — because the CSV-to-payload mapping only ever populates pdfUrls.original (masking happens downstream, after ingestion, per schema.sql's masked_pdf_urls/pii_masked_at columns). This is a known, currently-accepted gap in STORY_KAFKA_SCHEMA (confirmed with the team; schema intentionally left as-is for now) rather than a bug in the CSV pipeline itself.",Verified,"Upload+process tests/csv_uploads/valid_story.csv (or any fully-populated story row) -DB: SELECT meta_data->'schema_validation_errors' FROM csv_uploads WHERE id=; -> exactly one problem, ""'data.pdfUrls.masked' is missing""; rows_pushed=0" -UPLOAD-027,CSV Upload & Process API,Kafka broker unreachable during row push marks record on_hold and raises,Edge,Critical,A record has passed pre-publish schema validation for at least one row; Kafka broker stopped/unreachable,"Process a record whose row(s) would pass schema validation, while Kafka is down",confluent_kafka.Producer.flush()/produce() fails; csv_upload_repo.update_status sets status='on_hold' with stage='Kafka Publishing' in meta_data; the activity re-raises (Temporal will not silently report success),Not Tested,"1) Stop the Kafka broker +DB: SELECT meta_data->'schema_validation_errors' FROM csv_uploads WHERE id=; -> exactly one problem, ""'data.pdfUrls.masked' is missing""; rows_pushed=0" +UPLOAD-027,CSV Upload & Process API,Kafka broker unreachable during row push marks record on_hold and raises,Edge,Critical,A record has passed pre-publish schema validation for at least one row; Kafka broker stopped/unreachable,"Process a record whose row(s) would pass schema validation, while Kafka is down",confluent_kafka.Producer.flush()/produce() fails; csv_upload_repo.update_status sets status='on_hold' with stage='Kafka Publishing' in meta_data; the activity re-raises (Temporal will not silently report success),Verified,"1) Stop the Kafka broker 2) Process a record with at least one schema-valid row (would require adjusting STORY_KAFKA_SCHEMA or using a discussion CSV without the pdfUrls.masked gap) 3) DB: SELECT status, meta_data->>'stage' FROM csv_uploads WHERE id=; -> status='on_hold', stage='Kafka Publishing' 4) Temporal UI: confirm the csv_push_to_kafka_activity attempt shows a failure, not a false success -5) Restart Kafka afterward" -UPLOAD-028,CSV Upload & Process API,Missing program/leader-category DB match falls back to a generated UUID and generic name,Edge,Medium,program_name/leader_category values on the upload do not match any existing programs/leader_category row,Upload + process a CSV with a program_name/leader_category that has never been seen before (no matching DB row),"csv_processing_activity.py's DB lookup finds no match; leader_info/program_info fall back to {""id"": str(uuid.uuid4()), ""name"": , ...} rather than failing — the row is still processed (assuming it otherwise passes schema validation) with a freshly generated UUID standing in for the real id",Verified,"Code-level confirmation (app/temporal/csv_processing_activity.py lines ~201-213): when leader_row/program_row from the DB query is None, leader_info/program_info are built with id=str(uuid.uuid4()). To observe directly, use a program_name/leader_category guaranteed not to exist in the programs/leader_category tables and inspect the constructed payload (e.g. via a temporary debug log) to confirm tags.programId/tags.leaderCategoryId are freshly generated UUIDs, not matching any row in programs/leader_category." -UPLOAD-029,CSV Upload & Process API,CsvBatchProcessingWorkflow fans out all pending records as child workflows,Positive,High,PROCESSING_MODE=batch; multiple csv_uploads rows with status='pending',Manually start CsvBatchProcessingWorkflow (or wait for the csv-batch-processing schedule to fire),fetch_pending_csv_uploads_activity returns all pending ids; one CsvProcessingWorkflow child (id=csv-batch-child-) is started per pending record; results are aggregated into processed_count/success_count/failed_count,Not Tested,"1) Ensure 2+ csv_uploads rows have status='pending' +5) Restart Kafka afterward" +UPLOAD-028,CSV Upload & Process API,Missing program/leader-category DB match falls back to a generated UUID and generic name,Edge,Medium,program_name/leader_category values on the upload do not match any existing programs/leader_category row,Upload + process a CSV with a program_name/leader_category that has never been seen before (no matching DB row),"csv_processing_activity.py's DB lookup finds no match; leader_info/program_info fall back to {""id"": str(uuid.uuid4()), ""name"": , ...} rather than failing — the row is still processed (assuming it otherwise passes schema validation) with a freshly generated UUID standing in for the real id",Verified,"Code-level confirmation (app/temporal/csv_processing_activity.py lines ~201-213): when leader_row/program_row from the DB query is None, leader_info/program_info are built with id=str(uuid.uuid4()). To observe directly, use a program_name/leader_category guaranteed not to exist in the programs/leader_category tables and inspect the constructed payload (e.g. via a temporary debug log) to confirm tags.programId/tags.leaderCategoryId are freshly generated UUIDs, not matching any row in programs/leader_category." +UPLOAD-029,CSV Upload & Process API,CsvBatchProcessingWorkflow fans out all pending records as child workflows,Positive,High,PROCESSING_MODE=batch; multiple csv_uploads rows with status='pending',Manually start CsvBatchProcessingWorkflow (or wait for the csv-batch-processing schedule to fire),fetch_pending_csv_uploads_activity returns all pending ids; one CsvProcessingWorkflow child (id=csv-batch-child-) is started per pending record; results are aggregated into processed_count/success_count/failed_count,Verified,"1) Ensure 2+ csv_uploads rows have status='pending' 2) Start CsvBatchProcessingWorkflow manually via a Temporal client, or trigger the csv-batch-processing schedule 3) Temporal UI: confirm one csv-batch-child- child workflow per pending record -4) DB: confirm all previously-pending rows have moved to a terminal status" -UPLOAD-030,CSV Upload & Process API,CsvBatchProcessingWorkflow with zero pending records returns processed_count=0,Edge,Low,No csv_uploads rows with status='pending',Trigger CsvBatchProcessingWorkflow,"Returns {processed_count: 0, message: 'No pending CSV uploads found.'} without starting any child workflow",Not Tested,"1) DB: confirm SELECT count(*) FROM csv_uploads WHERE status='pending'; = 0 +4) DB: confirm all previously-pending rows have moved to a terminal status" +UPLOAD-030,CSV Upload & Process API,CsvBatchProcessingWorkflow with zero pending records returns processed_count=0,Edge,Low,No csv_uploads rows with status='pending',Trigger CsvBatchProcessingWorkflow,"Returns {processed_count: 0, message: 'No pending CSV uploads found.'} without starting any child workflow",Verified,"1) DB: confirm SELECT count(*) FROM csv_uploads WHERE status='pending'; = 0 2) Trigger CsvBatchProcessingWorkflow -3) Temporal UI: confirm the workflow result is {""processed_count"": 0, ""message"": ""No pending CSV uploads found.""}" +3) Temporal UI: confirm the workflow result is {""processed_count"": 0, ""message"": ""No pending CSV uploads found.""}" UPLOAD-031,CSV Upload & Process API,csv-batch-processing and daily-batch-processing schedules register when PROCESSING_MODE=batch,Positive,Medium,PROCESSING_MODE=batch,Start the Temporal worker with PROCESSING_MODE=batch,Both the 'csv-batch-processing' (cron CSV_SCHEDULE_CRON_TIME) and 'daily-batch-processing' (cron BATCH_SCHEDULE_CRON) schedules register successfully in Temporal on worker startup,Verified,"1) PROCESSING_MODE=batch python main.py --mode worker 2) Worker log should show 'CSV batch schedule successfully registered.' and 'Daily analysis batch schedule successfully registered.' -3) Temporal CLI/UI: confirm both schedule ids exist and show the correct cron expressions" +3) Temporal CLI/UI: confirm both schedule ids exist and show the correct cron expressions" UPLOAD-032,CSV Upload & Process API,Stale batch schedules are deleted when switching back to real-time,Positive,Medium,Both batch schedules from UPLOAD-031 are currently registered,Restart the Temporal worker with PROCESSING_MODE=real-time,Both 'csv-batch-processing' and 'daily-batch-processing' schedules are deleted on startup — prevents a schedule left behind from a prior batch-mode config from silently retrying forever with outdated arguments,Verified,"1) With both schedules registered (see UPLOAD-031), restart: python main.py --mode worker (PROCESSING_MODE=real-time, the .env default) 2) Worker log should show 'Deleted stale batch schedule 'csv-batch-processing' (PROCESSING_MODE=real-time).' and the same for 'daily-batch-processing' -3) Temporal CLI/UI: confirm neither schedule id exists anymore" +3) Temporal CLI/UI: confirm neither schedule id exists anymore" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0530de1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,336 @@ +""" +Shared test fakes/fixtures for tests/unit_testing.py. + +Everything here is a pure in-memory fake — no test in this suite ever +connects to a real Postgres, Kafka, Temporal, or GCS endpoint. The +conventions established in the (now-absorbed) test_mode_logic.py / +test_kafka_events.py are kept: plain unittest.mock (MagicMock/AsyncMock) +plus pytest's built-in monkeypatch fixture, async test bodies run via +asyncio.run() rather than pytest-asyncio. +""" +import csv +import json +import re +from pathlib import Path +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, MagicMock + + +# --------------------------------------------------------------------------- +# Fake asyncpg pool/connection — used for every db.pool.acquire() call site. +# --------------------------------------------------------------------------- + +class FakeConn: + """ + Stands in for an asyncpg.Connection. All query methods are AsyncMocks + with sane empty-result defaults; tests override .return_value/.side_effect + per case. Also usable as its own async context manager for conn.transaction(). + """ + + def __init__(self): + self.fetchrow = AsyncMock(return_value=None) + self.fetchval = AsyncMock(return_value=None) + self.fetch = AsyncMock(return_value=[]) + self.execute = AsyncMock(return_value="") + self.transaction = MagicMock(return_value=self) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + +class FakePool: + """Stands in for db.pool — .acquire() returns a FakeConn as an async context manager.""" + + def __init__(self, conn: Optional[FakeConn] = None): + self.conn = conn or FakeConn() + + def acquire(self): + return self.conn + + +def install_fake_db(monkeypatch, module, conn: Optional[FakeConn] = None) -> FakeConn: + """ + Patches `module.db.pool` with a FakePool wrapping `conn` (or a fresh FakeConn). + `module` is whichever module-under-test imported `db` (e.g. app.kafka.consumer, + app.temporal.pii_and_abusive_activity). Returns the FakeConn for assertions. + """ + fake_conn = conn or FakeConn() + monkeypatch.setattr(module.db, "pool", FakePool(fake_conn), raising=False) + return fake_conn + + +# --------------------------------------------------------------------------- +# Fake confluent_kafka Producer / AdminClient +# --------------------------------------------------------------------------- + +def make_fake_kafka_producer(delivery_error=None, flush_remaining: int = 0): + """ + A MagicMock standing in for confluent_kafka.Producer. `.produce()` invokes + the given callback immediately with `delivery_error` (None = success), + matching the real client's eventual-callback behavior closely enough for + the code under test (which only inspects the callback's error arg after + flush()). `.flush()` returns `flush_remaining` (0 = everything delivered). + """ + producer = MagicMock() + + def _produce(topic, value=None, key=None, headers=None, callback=None): + if callback: + callback(delivery_error, None) + + producer.produce = MagicMock(side_effect=_produce) + producer.poll = MagicMock(return_value=0) + producer.flush = MagicMock(return_value=flush_remaining) + return producer + + +def make_fake_admin_client(topics_exist: bool = True): + """ + A MagicMock standing in for confluent_kafka.admin.AdminClient. `.create_topics()` + returns a dict of topic name -> a future-like MagicMock whose `.result()` either + succeeds (topics_exist=False, i.e. freshly created) or raises a + "already exists"-style exception (topics_exist=True), matching how + consumer.py's `_ensure_topics_exist` treats both as non-fatal. + """ + def _create_topics(new_topics): + futures = {} + for nt in new_topics: + future = MagicMock() + if topics_exist: + future.result = MagicMock(side_effect=Exception("Topic already exists.")) + else: + future.result = MagicMock(return_value=None) + futures[nt.topic] = future + return futures + + client = MagicMock() + client.create_topics = MagicMock(side_effect=_create_topics) + return client + + +# --------------------------------------------------------------------------- +# Fake GCS storage.Client +# --------------------------------------------------------------------------- + +def make_fake_gcs_client(download_bytes: bytes = b""): + """ + A MagicMock standing in for google.cloud.storage.Client. Returns + (client, blob) so tests can assert on blob.upload_from_string / + upload_from_filename / download_as_bytes calls directly. + """ + blob = MagicMock() + blob.download_as_bytes = MagicMock(return_value=download_bytes) + bucket = MagicMock() + bucket.blob = MagicMock(return_value=blob) + client = MagicMock() + client.bucket = MagicMock(return_value=bucket) + return client, blob + + +# --------------------------------------------------------------------------- +# Fake LLM (OpenRouter) HTTP responses — patches urllib.request.urlopen +# --------------------------------------------------------------------------- + +class _FakeHTTPResponse: + def __init__(self, body: bytes): + self._body = body + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +def fake_llm_response(content: str, usage: Optional[Dict[str, Any]] = None): + """ + Builds a fake urllib.request.urlopen(...) context manager returning an + OpenRouter-shaped chat-completion JSON body, for patching + app.services.llm.urllib.request.urlopen. `content` can itself be malformed + JSON — that's the point for PII-003..006 / RATING-004/005 / THEME-011..015 + / SEC-003/004 style "staged bad LLM response" cases. + """ + body = { + "choices": [{"message": {"content": content}}], + "usage": usage if usage is not None else {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + return _FakeHTTPResponse(json.dumps(body).encode("utf-8")) + + +def install_fake_llm(monkeypatch, content: str, usage: Optional[Dict[str, Any]] = None): + """Patches app.services.llm.urllib.request.urlopen to return `content`/`usage`.""" + import app.services.llm as llm_module + response = fake_llm_response(content, usage) + monkeypatch.setattr(llm_module.urllib.request, "urlopen", MagicMock(return_value=response)) + + +def install_failing_llm(monkeypatch, exc: Exception): + """Patches urlopen to raise `exc` (simulating a network/HTTP failure).""" + import app.services.llm as llm_module + monkeypatch.setattr(llm_module.urllib.request, "urlopen", MagicMock(side_effect=exc)) + + +# --------------------------------------------------------------------------- +# Temporal workflow-context patching — lets CsvProcessingWorkflow.run() / +# CsvBatchProcessingWorkflow.run() / BatchProcessingWorkflow.run() / +# ConfigDrivenProcessingWorkflow.run() be called as plain coroutines with no +# real (or ephemeral) Temporal server at all. +# --------------------------------------------------------------------------- + +def install_fake_workflow_context(monkeypatch, activity_results: Optional[Dict[Any, Any]] = None, + child_workflow_results: Optional[Dict[Any, Any]] = None): + """ + Patches app.temporal.workflows.workflow.execute_activity/execute_child_workflow/ + now/continue_as_new/logger so workflow .run() methods can be invoked directly. + + `activity_results`: maps an activity function object -> the value + execute_activity should return for calls to that activity (or raise, if the + mapped value is an Exception instance). + `child_workflow_results`: same idea, keyed by child workflow .run method. + + Returns a MagicMock recording every execute_activity call (`.call_args_list`) + for call-count/ordering assertions, and a `continue_as_new_calls` list capturing + any continue_as_new(...) invocations (since it doesn't actually restart anything + here — the workflow body's `return`/loop-exit right after it, matching how the + real SDK never returns control past a continue_as_new call, is emulated by + raising a sentinel _ContinueAsNew exception the test can catch). + """ + import app.temporal.workflows as workflows_module + + activity_results = activity_results or {} + child_workflow_results = child_workflow_results or {} + continue_as_new_calls = [] + + async def fake_execute_activity(activity_fn, *args, **kwargs): + if activity_fn in activity_results: + result = activity_results[activity_fn] + if isinstance(result, Exception): + raise result + return result + return None + + async def fake_execute_child_workflow(run_fn, *args, **kwargs): + if run_fn in child_workflow_results: + result = child_workflow_results[run_fn] + if isinstance(result, Exception): + raise result + return result + return None + + class _ContinueAsNew(Exception): + pass + + def fake_continue_as_new(args=None, **kwargs): + continue_as_new_calls.append(args) + raise _ContinueAsNew() + + execute_activity_mock = AsyncMock(side_effect=fake_execute_activity) + monkeypatch.setattr(workflows_module.workflow, "execute_activity", execute_activity_mock) + monkeypatch.setattr(workflows_module.workflow, "execute_child_workflow", fake_execute_child_workflow) + monkeypatch.setattr(workflows_module.workflow, "now", MagicMock(return_value=__import__("datetime").datetime(2026, 1, 1))) + monkeypatch.setattr(workflows_module.workflow, "continue_as_new", fake_continue_as_new) + monkeypatch.setattr(workflows_module.workflow, "logger", MagicMock()) + + return execute_activity_mock, continue_as_new_calls, _ContinueAsNew + + +# --------------------------------------------------------------------------- +# Settings override helper +# --------------------------------------------------------------------------- + +def settings_override(monkeypatch, settings_obj, **overrides): + """Convenience wrapper for repeated monkeypatch.setattr(settings, k, v) calls.""" + for key, value in overrides.items(): + monkeypatch.setattr(settings_obj, key, value) + + +# --------------------------------------------------------------------------- +# TEST_CASES.csv sync — after a run of tests/unit_testing.py, reflect each +# test's pass/fail outcome back onto its Test ID row's Status column and +# print a one-line summary. Test function names follow the convention +# test__<3-digit-id>_ (e.g. test_kafka_020_... -> +# KAFKA-020). Four IDs are permanently excluded from this sync and keep +# whatever Status they already have in the sheet: KAFKA-021/BATCH-006 have +# only a narrower automated approximation (not a full validation of the +# documented behavior), and THEME-018/RATING-009 have no automated coverage +# at all (they need a live, concurrently-loaded system to observe) — all +# four are already Verified from manual QA. See TEST_CASES.csv's notes. +# --------------------------------------------------------------------------- + +_TEST_ID_RE = re.compile(r"^test_(kafka|sec|config|db|theme|pii|rating|batch|mode|llm|upload)_(\d{3})") +_STATUS_SYNC_EXCLUDE = {"KAFKA-021", "BATCH-006", "THEME-018", "RATING-009"} +_CSV_PATH = Path(__file__).parent / "TEST_CASES.csv" + +_test_outcomes: Dict[str, str] = {} # Test ID -> "passed" | "failed" | "skipped" + + +def pytest_runtest_logreport(report): + if report.when == "call": + outcome = report.outcome + elif report.when == "setup" and report.outcome in ("failed", "skipped"): + outcome = report.outcome + else: + return + + func_name = report.nodeid.split("::")[-1].split("[")[0] # strip parametrize suffix + match = _TEST_ID_RE.match(func_name) + if not match: + return + module, number = match.groups() + test_id = f"{module.upper()}-{number}" + + # A Test ID can be covered by more than one test function (e.g. + # MODE-003 / MODE-003b); any failure among them marks the ID failing. + if outcome == "failed" or _test_outcomes.get(test_id) == "failed": + _test_outcomes[test_id] = "failed" + elif test_id not in _test_outcomes: + _test_outcomes[test_id] = outcome + + +def pytest_sessionfinish(session, exitstatus): + if not _test_outcomes or not _CSV_PATH.exists(): + return + + with open(_CSV_PATH, newline="", encoding="utf-8") as f: + rows = list(csv.reader(f)) + header = rows[0] + id_idx = header.index("Test ID") + status_idx = header.index("Status") + + verified = failing = skipped = not_run = excluded = 0 + for row in rows[1:]: + test_id = row[id_idx] + if test_id in _STATUS_SYNC_EXCLUDE: + excluded += 1 + continue + outcome = _test_outcomes.get(test_id) + if outcome == "passed": + row[status_idx] = "Verified" + verified += 1 + elif outcome == "failed": + row[status_idx] = "Failing" + failing += 1 + elif outcome == "skipped": + skipped += 1 + else: + not_run += 1 # no test function ran for this ID this session — status left untouched + + with open(_CSV_PATH, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") + writer.writerows(rows) + + summary = ( + f"TEST_CASES.csv sync — {verified} verified, {failing} failing, " + f"{skipped} skipped, {not_run} not run this session, {excluded} excluded" + ) + terminal = session.config.pluginmanager.get_plugin("terminalreporter") + if terminal: + terminal.write_line("") + terminal.write_line(summary) + else: + print(summary) diff --git a/tests/test_kafka_events.py b/tests/test_kafka_events.py deleted file mode 100644 index 65ac5cb..0000000 --- a/tests/test_kafka_events.py +++ /dev/null @@ -1,71 +0,0 @@ -import asyncio -import json -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import json5 -import pytest - -from app.kafka import consumer as consumer_module - - -FIXTURE_ROOT = Path(__file__).resolve().parent / "kafka_events" - - -def _load_fixture(path: Path) -> dict: - text = path.read_text(encoding="utf-8") - return json5.loads(text) - - -class _FakeConn: - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - return False - - -class _FakePool: - def acquire(self): - return _FakeConn() - - -@pytest.mark.parametrize( - ("fixture_path", "expected_event_type"), - [ - (FIXTURE_ROOT / "create" / "create_discussion.json", "create"), - (FIXTURE_ROOT / "create" / "create_story.json", "create"), - (FIXTURE_ROOT / "update" / "update_discussion.json", "update"), - (FIXTURE_ROOT / "update" / "update_story.json", "update"), - (FIXTURE_ROOT / "delete" / "delete_discussion.json", "delete"), - (FIXTURE_ROOT / "delete" / "delete_story.json", "delete"), - ], -) -def test_process_message_routes_fixture_events(monkeypatch, fixture_path, expected_event_type): - event = _load_fixture(fixture_path) - payload = json.dumps(event) - - consumer = consumer_module.IngestionConsumer() - - insert_mock = AsyncMock() - delete_mock = AsyncMock() - trigger_mock = AsyncMock() - - monkeypatch.setattr(consumer_module.settings, "PROCESSING_MODE", "real-time", raising=False) - monkeypatch.setattr(consumer_module.db, "pool", SimpleNamespace(acquire=_FakePool().acquire), raising=False) - monkeypatch.setattr(consumer_module, "insert_or_update_submission", insert_mock, raising=False) - monkeypatch.setattr(consumer_module, "delete_submission", delete_mock, raising=False) - monkeypatch.setattr(consumer, "_trigger_realtime_workflow", trigger_mock, raising=False) - - asyncio.run(consumer.process_message(payload)) - - if expected_event_type in {"create", "update"}: - insert_mock.assert_awaited_once() - trigger_mock.assert_awaited_once_with(str(event["submissionId"]), event["tenantCode"], event["submissionType"]) - delete_mock.assert_not_awaited() - else: - assert delete_mock.await_count == 1 - assert delete_mock.await_args.args[1:] == (str(event["submissionId"]), event["tenantCode"]) - insert_mock.assert_not_awaited() - trigger_mock.assert_not_awaited() diff --git a/tests/test_mode_logic.py b/tests/test_mode_logic.py deleted file mode 100644 index 42e6053..0000000 --- a/tests/test_mode_logic.py +++ /dev/null @@ -1,177 +0,0 @@ -import asyncio -from unittest.mock import AsyncMock, patch, MagicMock, ANY -import pytest -from app.kafka import consumer as consumer_module -from app.temporal import worker as worker_module - -class _FakeConn: - async def __aenter__(self): - return self - async def __aexit__(self, exc_type, exc, tb): - return False - -class _FakePool: - def acquire(self): - return _FakeConn() - - -def test_trigger_realtime_workflow_connection_healing_success(monkeypatch): - """Test that if temporal_client is None, it heals connection and successfully triggers.""" - async def run_test(): - consumer = consumer_module.IngestionConsumer() - consumer.temporal_client = None # Start disconnected - - # Mock settings and DB pool - monkeypatch.setattr(consumer_module.settings, "PROCESSING_MODE", "real-time") - monkeypatch.setattr(consumer_module.db, "pool", MagicMock(acquire=_FakePool().acquire)) - - mock_update_status = AsyncMock() - monkeypatch.setattr(consumer_module, "update_submission_status", mock_update_status) - - mock_client = MagicMock() - mock_client.start_workflow = AsyncMock() - - # Mock Client.connect to return our mock_client - with patch("app.kafka.consumer.Client.connect", AsyncMock(return_value=mock_client)) as mock_connect: - await consumer._trigger_realtime_workflow("sub1", "tenant1", "story") - - mock_connect.assert_awaited_once_with("localhost:7233") - mock_client.start_workflow.assert_awaited_once() - mock_update_status.assert_awaited_once_with( - ANY, - "sub1", "tenant1", "processing" - ) - assert consumer.temporal_client is mock_client - - asyncio.run(run_test()) - - -def test_trigger_realtime_workflow_connection_healing_failure(monkeypatch): - """Test that if connection healing fails, it logs error and does not raise exception.""" - async def run_test(): - consumer = consumer_module.IngestionConsumer() - consumer.temporal_client = None - - monkeypatch.setattr(consumer_module.settings, "PROCESSING_MODE", "real-time") - monkeypatch.setattr(consumer_module.db, "pool", MagicMock(acquire=_FakePool().acquire)) - - mock_update_status = AsyncMock() - monkeypatch.setattr(consumer_module, "update_submission_status", mock_update_status) - - # Client.connect raises exception - with patch("app.kafka.consumer.Client.connect", AsyncMock(side_effect=Exception("Connection refused"))): - await consumer._trigger_realtime_workflow("sub1", "tenant1", "story") - - # Should not update status to processing - mock_update_status.assert_not_awaited() - assert consumer.temporal_client is None - - asyncio.run(run_test()) - - -def test_trigger_realtime_workflow_grpc_error_resets_client(monkeypatch): - """Test that a gRPC connection error during start_workflow resets client to None.""" - async def run_test(): - consumer = consumer_module.IngestionConsumer() - mock_client = MagicMock() - # Simulate a gRPC unavailable/connection error - mock_client.start_workflow = AsyncMock(side_effect=Exception("gRPC status: UNAVAILABLE, description: connection lost")) - consumer.temporal_client = mock_client - - monkeypatch.setattr(consumer_module.settings, "PROCESSING_MODE", "real-time") - monkeypatch.setattr(consumer_module.db, "pool", MagicMock(acquire=_FakePool().acquire)) - - mock_update_status = AsyncMock() - monkeypatch.setattr(consumer_module, "update_submission_status", mock_update_status) - - await consumer._trigger_realtime_workflow("sub1", "tenant1", "story") - - # Client should be reset to None for future healing - assert consumer.temporal_client is None - mock_update_status.assert_not_awaited() - - asyncio.run(run_test()) - - -def test_worker_startup_registers_batch_schedule(monkeypatch): - """Test that start_worker registers daily batch schedule when mode is batch.""" - async def run_test(): - monkeypatch.setattr(worker_module.settings, "PROCESSING_MODE", "batch") - monkeypatch.setattr(worker_module.settings, "BATCH_SCHEDULE_CRON", "0 20 * * *") - monkeypatch.setattr(worker_module.db, "connect", AsyncMock()) - monkeypatch.setattr(worker_module.db, "disconnect", AsyncMock()) - - mock_client = MagicMock() - mock_client.create_schedule = AsyncMock() - - monkeypatch.setattr(worker_module.Client, "connect", AsyncMock(return_value=mock_client)) - - # Mock Worker class run to raise CancelledError immediately so it exits - mock_worker_instance = MagicMock() - mock_worker_instance.run = AsyncMock(side_effect=asyncio.CancelledError()) - with patch("app.temporal.worker.Worker", return_value=mock_worker_instance): - await worker_module.start_worker() - - mock_client.create_schedule.assert_awaited_once() - # Verify schedule spec cron expression is correct - args, kwargs = mock_client.create_schedule.call_args - assert kwargs["id"] == "daily-batch-processing" - assert kwargs["schedule"].spec.cron_expressions == ["0 20 * * *"] - - asyncio.run(run_test()) - - -def test_deface_blur_activity_success(monkeypatch): - """Test that deface_blur_activity downloads, processes, uploads, and deletes temp files successfully.""" - async def run_test(): - from app.temporal.deface_blur_activity import deface_blur_activity - from app.temporal.deface_blur_activity import db as db_module - from pathlib import Path - - # Mock db connection acquire - mock_conn = AsyncMock() - mock_conn.execute = AsyncMock() - - class _CustomFakeConn: - async def __aenter__(self): - return mock_conn - async def __aexit__(self, exc_type, exc, tb): - return False - - monkeypatch.setattr(db_module, "pool", MagicMock(acquire=lambda: _CustomFakeConn())) - - # Mock payload helper - monkeypatch.setattr( - "app.temporal.deface_blur_activity.get_submission_type_and_payload", - AsyncMock(return_value=("story", {"image_urls": ["https://foo.com/3281/bar.png"]})) - ) - - # Mock download/blur/upload - mock_download = MagicMock(return_value=Path("/tmp/bar.png")) - monkeypatch.setattr("app.temporal.deface_blur_activity._download_file", mock_download) - - mock_anonymize = MagicMock() - monkeypatch.setattr("app.temporal.deface_blur_activity.anonymize_face", mock_anonymize) - - mock_upload = MagicMock(return_value="/bucket/story_blurred_image/3281/bar.png") - monkeypatch.setattr("app.temporal.deface_blur_activity.upload_to_gcp", mock_upload) - - # Mock path exists and unlink - mock_path = MagicMock() - mock_path.__truediv__.return_value = mock_path - mock_path.exists.return_value = True - mock_path.unlink = MagicMock() - - monkeypatch.setattr("app.temporal.deface_blur_activity.DOWNLOADS_DIR", mock_path) - monkeypatch.setattr("app.temporal.deface_blur_activity.OUTPUTS_DIR", mock_path) - - res = await deface_blur_activity({"submission_id": "sub1", "tenant_code": "tenant1"}) - - assert res["status"] == "success" - assert res["blur_paths"] == ["/bucket/story_blurred_image/3281/bar.png"] - mock_conn.execute.assert_called_once() - # Verify cleanups were called on both local_path and output_path - assert mock_path.unlink.call_count == 2 - - asyncio.run(run_test()) - diff --git a/tests/unit_testing.py b/tests/unit_testing.py new file mode 100644 index 0000000..f44421d --- /dev/null +++ b/tests/unit_testing.py @@ -0,0 +1,2220 @@ +""" +Comprehensive unit-test suite covering tests/TEST_CASES.csv. + +No real Postgres, Kafka, or Temporal connection is ever made — every external +dependency (asyncpg pool, confluent_kafka Producer/Consumer/AdminClient, +google.cloud.storage.Client, urllib-based LLM calls, and Temporal's +Client/workflow context) is mocked via conftest.py's shared fakes plus plain +unittest.mock, following this repo's existing convention (no pytest-asyncio — +async bodies run via asyncio.run()). + +Sections mirror tests/TEST_CASES.csv's module column, in the same order: + KAFKA-*, SEC-*, CONFIG-*, DB-*, THEME-*, PII-*, RATING-*, BATCH-*, MODE-*, + LLM-*, UPLOAD-* + +Explicitly excluded (documented, not attempted — see the plan this file was +built from): RATING-009, THEME-018, BATCH-006, and KAFKA-021 is narrowed to +an assertable subset. These are inherently live-system/timing/concurrency +observations, or (BATCH-006) enforced by Temporal's own server, not this +codebase. +""" +import asyncio +import json +import time +from pathlib import Path +from unittest.mock import ANY, AsyncMock, MagicMock + +import json5 +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.config import settings +from conftest import ( + FakeConn, + FakePool, + install_fake_db, + make_fake_kafka_producer, + make_fake_admin_client, + make_fake_gcs_client, + install_fake_llm, + install_failing_llm, + install_fake_workflow_context, + settings_override, +) + +FIXTURE_ROOT = Path(__file__).resolve().parent / "kafka_events" +CSV_FIXTURE_ROOT = Path(__file__).resolve().parent / "csv_uploads" + + +def _load_fixture(path: Path) -> dict: + return json5.loads(path.read_text(encoding="utf-8")) + + +def _fixture(*parts) -> dict: + return _load_fixture(FIXTURE_ROOT.joinpath(*parts)) + + +def _find_execute_call(conn, *needles): + """ + insert_or_update_submission's conn also runs tenant/leader_category/programs + upserts (and participant-metrics writes) via conn.execute before/after the + story_submissions or discussion_submissions upsert, so the *last* execute + call isn't reliably the one under test. Finds the one call.args[0] (the + SQL string) contains all of `needles`. + """ + matches = [c for c in conn.execute.call_args_list if all(n in c.args[0] for n in needles)] + assert len(matches) == 1, f"expected exactly one execute() call matching {needles}, found {len(matches)}" + return matches[0] + + +# ============================================================================= +# KAFKA INGESTION (KAFKA-*) +# ============================================================================= + +import app.kafka.consumer as consumer_module + + +def _consumer_with_mocks(monkeypatch, fetchval_return=None): + """A fresh IngestionConsumer with insert/delete/trigger mocked and a FakeConn + installed for db.pool (fetchval_return controls the create-duplicate check).""" + consumer = consumer_module.IngestionConsumer() + fake_conn = install_fake_db(monkeypatch, consumer_module) + fake_conn.fetchval.return_value = fetchval_return + insert_mock = AsyncMock(return_value={"id": "uuid-1", "status": "pending"}) + delete_mock = AsyncMock(return_value=True) + trigger_mock = AsyncMock() + monkeypatch.setattr(consumer_module, "insert_or_update_submission", insert_mock, raising=False) + monkeypatch.setattr(consumer_module, "delete_submission", delete_mock, raising=False) + monkeypatch.setattr(consumer, "_trigger_realtime_workflow", trigger_mock, raising=False) + monkeypatch.setattr(consumer_module.settings, "PROCESSING_MODE", "real-time", raising=False) + return consumer, insert_mock, delete_mock, trigger_mock + + +def test_kafka_001_valid_discussion_create_ingested(monkeypatch): + async def run_test(): + consumer, insert_mock, delete_mock, trigger_mock = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + await consumer.process_message(json.dumps(event)) + insert_mock.assert_awaited_once() + trigger_mock.assert_awaited_once_with(str(event["submissionId"]), event["tenantCode"], event["submissionType"]) + delete_mock.assert_not_awaited() + dlq_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_002_valid_story_create_ingested(monkeypatch): + async def run_test(): + consumer, insert_mock, delete_mock, trigger_mock = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_story.json") + await consumer.process_message(json.dumps(event)) + insert_mock.assert_awaited_once() + trigger_mock.assert_awaited_once() + dlq_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_003_update_applies_delta_only_newvalues(): + """Calls insert_or_update_submission directly (unmocked) to confirm fields + absent from newValues are passed as None, so the real SQL's COALESCE would + preserve them rather than overwrite with a stale/None value.""" + from app.database.operations import insert_or_update_submission + + async def run_test(): + conn = FakeConn() + conn.fetchrow.return_value = {"id": "uuid-1", "status": "processing"} + conn.fetchval.return_value = 1 # row_exists = True -> UPDATE branch + event = _fixture("update", "update_discussion.json") + await insert_or_update_submission(conn, event) + + update_call = _find_execute_call(conn, "UPDATE discussion_submissions") + args = update_call.args + # UPDATE discussion_submissions SET title=$3, challenges=$4, solutions=$5, ... + # newValues only has title/participantsData -> challenges/solutions args must be None + assert args[3] == event["newValues"]["title"] + assert args[4] is None # challenges + assert args[5] is None # solutions + asyncio.run(run_test()) + + +def test_kafka_004_delete_event_removes_submission(monkeypatch): + async def run_test(): + consumer, insert_mock, delete_mock, trigger_mock = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("delete", "delete_discussion.json") + await consumer.process_message(json.dumps(event)) + delete_mock.assert_awaited_once_with(ANY, str(event["submissionId"]), event["tenantCode"]) + insert_mock.assert_not_awaited() + trigger_mock.assert_not_awaited() + dlq_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_005_malformed_json_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + await consumer.process_message("{bad json") + dlq_mock.assert_awaited_once() + assert "Invalid JSON:" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +@pytest.mark.parametrize("payload,expected_type", [("[]", "list"), ("42", "int")]) +def test_kafka_006_non_object_json_routed_to_dlq(monkeypatch, payload, expected_type): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + await consumer.process_message(payload) + dlq_mock.assert_awaited_once() + assert f"got {expected_type}" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_007_missing_submission_id_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + del event["submissionId"] + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'submissionId' is missing" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_008_null_submission_id_routed_to_dlq_not_coerced(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["submissionId"] = None + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'submissionId' is null" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_009_missing_tenant_code_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + del event["tenantCode"] + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'tenantCode' is missing" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_010_empty_tags_state_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["tags"]["state"] = "" + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'tags.state' is empty" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_011_empty_challenges_array_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["data"]["challenges"] = [] + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'data.challenges' is empty" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_012_empty_solutions_array_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["data"]["solutions"] = [] + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'data.solutions' is empty" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +@pytest.mark.parametrize("bad_event_type,expected_fragment", [(123, "got int"), (None, "got NoneType")]) +def test_kafka_013_non_string_event_type_does_not_crash(monkeypatch, bad_event_type, expected_fragment): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["eventType"] = bad_event_type + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert expected_fragment in dlq_mock.await_args.args[1] + + # Consumer keeps working afterward — no crash/hang. + good_event = _fixture("create", "create_story.json") + await consumer.process_message(json.dumps(good_event)) + insert_mock.assert_awaited_once() + asyncio.run(run_test()) + + +def test_kafka_014_non_string_submission_type_does_not_crash(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["submissionType"] = 123 + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "Unrecognized submissionType 123" in dlq_mock.await_args.args[1] + + good_event = _fixture("create", "create_story.json") + await consumer.process_message(json.dumps(good_event)) + insert_mock.assert_awaited_once() + asyncio.run(run_test()) + + +def test_kafka_015_unrecognized_submission_type_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["submissionType"] = "survey" + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "Unrecognized submissionType 'survey'" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_016_unsupported_event_type_routed_to_dlq(monkeypatch): + """ + Note: validate_ingestion_schema() rejects ANY eventType outside + create/update/delete at the schema-lookup stage (event_schema.get(event_type) + is always None for "archive"), so process_message's final + `else: "Unsupported eventType: ..."` branch is never actually reached for + this input — it's effectively dead code given the current validator. This + test asserts the REAL reachable reason rather than the sheet's original + (unreachable) wording. + """ + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + event["eventType"] = "archive" + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "No ingestion schema section defined for eventType 'archive'" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_017_update_newvalues_empty_field_routed_to_dlq(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("update", "update_discussion.json") + event["newValues"]["title"] = "" + await consumer.process_message(json.dumps(event)) + dlq_mock.assert_awaited_once() + assert "'newValues.title' is empty" in dlq_mock.await_args.args[1] + insert_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_kafka_018_update_omitting_field_preserves_existing_value(): + """update_discussion.json's newValues omits challenges/solutions entirely — + the validator must not flag the omission (only newValuesNoEmpty checks keys + actually PRESENT in newValues), and the DB call must pass None for them.""" + from app.database.operations import insert_or_update_submission + from app.services.ingestion_validation import validate_ingestion_schema + + event = _fixture("update", "update_discussion.json") + problems = validate_ingestion_schema(event, event["submissionType"], "update") + assert problems == [] + + async def run_test(): + conn = FakeConn() + conn.fetchrow.return_value = {"id": "uuid-1", "status": "processing"} + conn.fetchval.return_value = 1 + await insert_or_update_submission(conn, event) + args = _find_execute_call(conn, "UPDATE discussion_submissions").args + assert args[4] is None # challenges + assert args[5] is None # solutions + asyncio.run(run_test()) + + +def test_kafka_019_duplicate_create_skipped(monkeypatch, caplog): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch, fetchval_return=1) # row already exists + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_story.json") + with caplog.at_level("WARNING"): + await consumer.process_message(json.dumps(event)) + insert_mock.assert_not_awaited() + dlq_mock.assert_not_awaited() + assert any("Duplicate entry" in r.message for r in caplog.records) + asyncio.run(run_test()) + + +class _FakeKafkaMsg: + def __init__(self, value: bytes): + self._value = value + + def error(self): + return None + + def value(self): + return self._value + + +def _make_fake_kafka_consumer(messages): + """poll() pops one message per call, then returns None forever (with a tiny + real sleep to keep the busy-loop from pegging a CPU core during the test).""" + remaining = list(messages) + + def _poll(timeout=1.0): + if remaining: + return remaining.pop(0) + time.sleep(0.01) + return None + + fake = MagicMock() + fake.poll = MagicMock(side_effect=_poll) + fake.commit = MagicMock() + fake.close = MagicMock() + fake.subscribe = MagicMock() + return fake + + +async def _run_consumer_briefly(consumer, duration=0.25): + task = asyncio.create_task(consumer.start()) + await asyncio.sleep(duration) + consumer.running = False + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +def test_kafka_020_db_failure_retries_3_times_then_dlq(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + # No asyncio.sleep patch here — consumer_module.asyncio *is* the real + # asyncio module, so patching .sleep globally would also break this + # test's own timing helper. The real 2s/4s backoff is allowed to elapse. + monkeypatch.setattr(consumer_module, "Producer", MagicMock(return_value=make_fake_kafka_producer())) + monkeypatch.setattr(consumer_module, "AdminClient", MagicMock(return_value=make_fake_admin_client())) + monkeypatch.setattr(consumer_module, "Consumer", MagicMock( + return_value=_make_fake_kafka_consumer([_FakeKafkaMsg(b'{"submissionId": 1}')]) + )) + monkeypatch.setattr(consumer_module.db, "connect", AsyncMock()) + monkeypatch.setattr(consumer_module.db, "disconnect", AsyncMock()) + monkeypatch.setattr(consumer_module.Client, "connect", AsyncMock(side_effect=Exception("no temporal"))) + + process_mock = AsyncMock(side_effect=RuntimeError("session_id collision")) + monkeypatch.setattr(consumer, "process_message", process_mock) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + + await _run_consumer_briefly(consumer, duration=7.0) # 2s + 4s real backoff + margin + + assert process_mock.await_count == 3 + dlq_mock.assert_awaited_once() + assert "Processing failed after 3 attempts" in dlq_mock.await_args.args[1] + asyncio.run(run_test()) + + +def test_kafka_021_dlq_failure_leaves_offset_uncommitted(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + # Real 2s/4s backoff is allowed to elapse (see test_kafka_020 comment). + fake_kafka_consumer = _make_fake_kafka_consumer([_FakeKafkaMsg(b'{"submissionId": 1}')]) + monkeypatch.setattr(consumer_module, "Producer", MagicMock(return_value=make_fake_kafka_producer())) + monkeypatch.setattr(consumer_module, "AdminClient", MagicMock(return_value=make_fake_admin_client())) + monkeypatch.setattr(consumer_module, "Consumer", MagicMock(return_value=fake_kafka_consumer)) + monkeypatch.setattr(consumer_module.db, "connect", AsyncMock()) + monkeypatch.setattr(consumer_module.db, "disconnect", AsyncMock()) + monkeypatch.setattr(consumer_module.Client, "connect", AsyncMock(side_effect=Exception("no temporal"))) + + monkeypatch.setattr(consumer, "process_message", AsyncMock(side_effect=RuntimeError("boom"))) + monkeypatch.setattr(consumer, "_send_to_dlq", AsyncMock(side_effect=RuntimeError("dlq unreachable"))) + + await _run_consumer_briefly(consumer, duration=7.0) + + fake_kafka_consumer.commit.assert_not_called() + asyncio.run(run_test()) + + +def test_kafka_022_delimiter_character_preserved_intact(): + """A statement containing a literal '|' round-trips as one unmodified TEXT[] + element — _normalize_statement_list never delimiter-splits.""" + from app.database.operations import _normalize_statement_list + + value = ["Space is an issue | teachers agree"] + result = _normalize_statement_list(value) + assert result == ["Space is an issue | teachers agree"] + assert len(result) == 1 + + +def test_kafka_023_later_message_never_commits_past_unresolved_earlier(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + # Real 2s/4s backoff is allowed to elapse (see test_kafka_020 comment). + fake_kafka_consumer = _make_fake_kafka_consumer([ + _FakeKafkaMsg(b'{"submissionId": "fail-me"}'), + _FakeKafkaMsg(b'{"submissionId": "succeed-me"}'), + ]) + monkeypatch.setattr(consumer_module, "Producer", MagicMock(return_value=make_fake_kafka_producer())) + monkeypatch.setattr(consumer_module, "AdminClient", MagicMock(return_value=make_fake_admin_client())) + monkeypatch.setattr(consumer_module, "Consumer", MagicMock(return_value=fake_kafka_consumer)) + monkeypatch.setattr(consumer_module.db, "connect", AsyncMock()) + monkeypatch.setattr(consumer_module.db, "disconnect", AsyncMock()) + monkeypatch.setattr(consumer_module.Client, "connect", AsyncMock(side_effect=Exception("no temporal"))) + + call_order = [] + + async def fake_process(raw_payload): + payload = json.loads(raw_payload) + if payload["submissionId"] == "fail-me": + call_order.append("process:fail-me") + raise RuntimeError("boom") + call_order.append("process:succeed-me") + + monkeypatch.setattr(consumer, "process_message", fake_process) + + async def fake_dlq(raw_payload, reason, identifiers=None): + call_order.append("dlq:fail-me") + + monkeypatch.setattr(consumer, "_send_to_dlq", fake_dlq) + + await _run_consumer_briefly(consumer, duration=7.0) + + # All 3 retries + the DLQ publish for the failing message happen before + # the second message is ever handed to process_message. + first_success_index = call_order.index("process:succeed-me") + assert call_order[:first_success_index].count("process:fail-me") == 3 + assert "dlq:fail-me" in call_order[:first_success_index] + asyncio.run(run_test()) + + +def test_kafka_024_topics_auto_created_on_startup(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + admin_client = make_fake_admin_client(topics_exist=False) + monkeypatch.setattr(consumer_module, "AdminClient", MagicMock(return_value=admin_client)) + await consumer._ensure_topics_exist(["analytics.ingestion.raw", "analytics.ingestion.raw.dlq"]) + admin_client.create_topics.assert_called_once() + created_topics = [nt.topic for nt in admin_client.create_topics.call_args.args[0]] + assert set(created_topics) == {"analytics.ingestion.raw", "analytics.ingestion.raw.dlq"} + asyncio.run(run_test()) + + +def test_kafka_025_offset_commits_only_after_success(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + fake_kafka_consumer = _make_fake_kafka_consumer([_FakeKafkaMsg(b'{"submissionId": 1}')]) + monkeypatch.setattr(consumer_module, "Producer", MagicMock(return_value=make_fake_kafka_producer())) + monkeypatch.setattr(consumer_module, "AdminClient", MagicMock(return_value=make_fake_admin_client())) + monkeypatch.setattr(consumer_module, "Consumer", MagicMock(return_value=fake_kafka_consumer)) + monkeypatch.setattr(consumer_module.db, "connect", AsyncMock()) + monkeypatch.setattr(consumer_module.db, "disconnect", AsyncMock()) + monkeypatch.setattr(consumer_module.Client, "connect", AsyncMock(side_effect=Exception("no temporal"))) + monkeypatch.setattr(consumer, "process_message", AsyncMock()) # succeeds + + await _run_consumer_briefly(consumer) + + fake_kafka_consumer.commit.assert_called_once() + assert fake_kafka_consumer.commit.call_args.kwargs.get("asynchronous") is False + asyncio.run(run_test()) + + +# ============================================================================= +# SECURITY & LOGGING (SEC-*) +# ============================================================================= + +def test_sec_001_raw_payload_never_logged_in_plaintext(monkeypatch, caplog): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + dlq_mock = AsyncMock() + monkeypatch.setattr(consumer, "_send_to_dlq", dlq_mock) + event = _fixture("create", "create_discussion.json") + marker = "UNIQUE_MARKER_TEXT" + event["data"]["challenges"] = [marker] + event["tags"]["state"] = "" # force a validation failure -> DLQ path logs the payload fingerprint + with caplog.at_level("ERROR"): + await consumer.process_message(json.dumps(event)) + assert not any(marker in r.message for r in caplog.records) + asyncio.run(run_test()) + + +def test_sec_002_dlq_headers_surface_only_non_sensitive_identifiers(monkeypatch): + async def run_test(): + consumer, insert_mock, _, _ = _consumer_with_mocks(monkeypatch) + producer = make_fake_kafka_producer() + consumer.dlq_producer = producer + event = _fixture("create", "create_discussion.json") + event["tags"]["state"] = "" + await consumer.process_message(json.dumps(event)) + produce_call = producer.produce.call_args + headers = dict(produce_call.kwargs["headers"]) + assert set(headers.keys()) <= {"reason", "submissionId", "tenantCode", "sessionId"} + assert headers["submissionId"] == str(event["submissionId"]).encode("utf-8") + asyncio.run(run_test()) + + +def test_sec_003_pii_malformed_llm_response_never_logs_full_content(monkeypatch, caplog): + async def run_test(): + import app.temporal.pii_and_abusive_activity as pii_module + conn = install_fake_db(monkeypatch, pii_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{text}}"} + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "story", {"objective": "some text"} + + monkeypatch.setattr(pii_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + monkeypatch.setattr(operations_module, "update_submission_status", AsyncMock()) + monkeypatch.setattr(pii_module, "insert_llm_log", AsyncMock()) + + marker = "FAKE_PII_MARKER_12345" + install_fake_llm(monkeypatch, content=f"not json at all {marker}") + + with caplog.at_level("ERROR"): + with pytest.raises(Exception): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + assert any("Failed to parse LLM response JSON" in r.message for r in caplog.records) + assert not any(marker in r.message for r in caplog.records) + asyncio.run(run_test()) + + +def test_sec_004_thematic_malformed_llm_response_never_logs_full_content(monkeypatch, caplog): + async def run_test(): + import app.temporal.thematic_activity as thematic_module + marker = "FAKE_PII_MARKER_67890" + install_fake_llm(monkeypatch, content=f"not json {marker}") + conn = install_fake_db(monkeypatch, thematic_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + monkeypatch.setattr(thematic_module, "insert_llm_log", AsyncMock()) + + with caplog.at_level("ERROR"): + with pytest.raises(Exception): + await thematic_module._run_batched_llm_fallback( + pending_items=[{"statement": "x", "statement_type": "challenges", "is_discussion": True, + "best_similarity": 0.1, "diagnostics": {}}], + approved_themes=[], theme_id_to_info={}, + submission_id="1", tenant_code="mitra", analysis_type="thematic_classification", + resolved_model="m", resolved_max_tokens=100, resolved_timeout=10, + ) + assert any("JSON parsing failed for batched LLM response" in r.message for r in caplog.records) + assert not any(marker in r.message for r in caplog.records) + asyncio.run(run_test()) + + +# ============================================================================= +# CONFIG & SETTINGS (CONFIG-*) +# ============================================================================= + +from app.config import Settings + + +def _settings_kwargs(**overrides): + base = dict(AUTH_TOKEN="test-token") + base.update(overrides) + return base + + +def test_config_001_valid_process_config_loads(): + s = Settings(**_settings_kwargs()) + assert s.get_process_config("story")[0]["name"] == "pii_and_abusive_language_detection" + assert isinstance(s.get_process_config("discussion"), list) + + +def test_config_002_invalid_process_config_story_fails_fast(): + with pytest.raises(Exception, match="Invalid JSON configuration for PROCESS_CONFIG_STORY"): + Settings(**_settings_kwargs(PROCESS_CONFIG_STORY="not valid json")) + + +def test_config_003_valid_kafka_schema_loads(): + s = Settings(**_settings_kwargs()) + assert set(s.get_kafka_ingestion_schema("story").keys()) == {"create", "update", "delete"} + assert set(s.get_kafka_ingestion_schema("discussion").keys()) == {"create", "update", "delete"} + + +def test_config_004_kafka_schema_missing_required_key_fails_validation(): + bad_schema = json.dumps({"create": {"required": []}, "update": {"required": []}}) # no "delete" + with pytest.raises(Exception, match="must be a JSON object with 'create', 'update', and 'delete' keys"): + Settings(**_settings_kwargs(STORY_KAFKA_SCHEMA=bad_schema)) + + +def test_config_005_kafka_schema_required_not_list_of_strings_fails_validation(): + bad_schema = json.dumps({ + "create": {"required": "not-a-list"}, + "update": {"required": []}, + "delete": {"required": []}, + }) + with pytest.raises(Exception, match="must be a list of strings"): + Settings(**_settings_kwargs(DISCUSSION_KAFKA_SCHEMA=bad_schema)) + + +def test_config_006_reset_db_with_production_environment_is_refused(monkeypatch, caplog): + async def run_test(): + import app.database.db as db_module + settings_override(monkeypatch, db_module.settings, RESET_DB=True, ENVIRONMENT="production") + fake_conn = FakeConn() + db_instance = db_module.Database() + db_instance.pool = FakePool(fake_conn) + with caplog.at_level("ERROR"): + await db_instance.initialize_schema() + assert not any("DROP SCHEMA" in str(c.args[0]) for c in fake_conn.execute.call_args_list if c.args) + assert any("refusing to drop the schema" in r.message for r in caplog.records) + asyncio.run(run_test()) + + +def test_config_007_reset_db_with_development_environment_resets_schema(monkeypatch): + async def run_test(): + import app.database.db as db_module + settings_override(monkeypatch, db_module.settings, RESET_DB=True, ENVIRONMENT="development") + fake_conn = FakeConn() + db_instance = db_module.Database() + db_instance.pool = FakePool(fake_conn) + await db_instance.initialize_schema() + executed = [c.args[0] for c in fake_conn.execute.call_args_list if c.args] + assert any("DROP SCHEMA IF EXISTS public CASCADE" in stmt for stmt in executed) + assert any("CREATE SCHEMA public" in stmt for stmt in executed) + asyncio.run(run_test()) + + +def test_config_008_unrecognized_submission_type_returns_empty_process_config(): + s = Settings(**_settings_kwargs()) + assert s.get_process_config("survey") == [] + + +def test_config_009_unrecognized_submission_type_raises_for_kafka_schema(): + s = Settings(**_settings_kwargs()) + with pytest.raises(ValueError, match="No Kafka ingestion schema defined"): + s.get_kafka_ingestion_schema(None) + + +# ============================================================================= +# DATABASE OPERATIONS (DB-*) +# ============================================================================= + +from app.database.operations import ( + _normalize_statement_list, + insert_or_update_submission, +) + + +def test_db_001_insert_discussion_stores_text_array(): + async def run_test(): + conn = FakeConn() + conn.fetchrow.return_value = {"id": "uuid-1", "status": "pending"} + conn.fetchval.return_value = None # row_exists = False -> INSERT branch + event = _fixture("create", "create_discussion.json") + await insert_or_update_submission(conn, event) + insert_call = _find_execute_call(conn, "INSERT INTO discussion_submissions") + challenges_arg = insert_call.args[4] + assert isinstance(challenges_arg, list) + assert len(challenges_arg) == len(event["data"]["challenges"]) + asyncio.run(run_test()) + + +def test_db_002_insert_story_stores_scalar_text(): + async def run_test(): + conn = FakeConn() + conn.fetchrow.return_value = {"id": "uuid-1", "status": "pending"} + conn.fetchval.return_value = None + event = _fixture("create", "create_story.json") + await insert_or_update_submission(conn, event) + insert_call = conn.execute.call_args_list[-1] + objective_arg = insert_call.args[4] + assert isinstance(objective_arg, str) + asyncio.run(run_test()) + + +def test_db_003_normalize_statement_list_wraps_single_string(): + assert _normalize_statement_list("a single statement") == ["a single statement"] + + +def test_db_004_normalize_statement_list_none_stays_none(): + assert _normalize_statement_list(None) is None + + +def test_db_005_update_omitting_masked_fields_does_not_revert_them(): + """An UPDATE whose newValues omits challenges/solutions must pass None for + them (COALESCE preserves whatever masked text is already in the DB), even + though oldValues (the producer's original unmasked text) is present.""" + async def run_test(): + conn = FakeConn() + conn.fetchrow.return_value = {"id": "uuid-1", "status": "processing"} + conn.fetchval.return_value = 1 + event = _fixture("update", "update_discussion.json") + assert "challenges" not in event["newValues"] + await insert_or_update_submission(conn, event) + args = _find_execute_call(conn, "UPDATE discussion_submissions").args + assert args[4] is None + assert args[5] is None + asyncio.run(run_test()) + + +def test_db_006_duplicate_session_id_raises_clear_error(): + import asyncpg + + async def run_test(): + conn = FakeConn() + + class _FakeUniqueViolation(asyncpg.exceptions.UniqueViolationError): + def __init__(self): + self.constraint_name = "submissions_session_id_key" + + conn.fetchrow.side_effect = _FakeUniqueViolation() + event = _fixture("create", "create_story.json") + with pytest.raises(ValueError, match="is already associated with a different submission"): + await insert_or_update_submission(conn, event) + asyncio.run(run_test()) + + +# ============================================================================= +# THEMATIC CLASSIFICATION (THEME-*) +# ============================================================================= +# THEME-018 excluded: "SentenceTransformer calls run off the event loop" is a +# live concurrent-progress observation, not assertable via mocks. + +import app.temporal.thematic_activity as thematic_module + + +def test_theme_001_short_statement_classified_unknown_unclear(monkeypatch): + async def run_test(): + conn = FakeConn() + insert_mock = AsyncMock() + monkeypatch.setattr(thematic_module, "insert_analysis_result", insert_mock) + result, pending = await thematic_module._run_local_classification( + conn=conn, statement="Too short", submission_id="1", tenant_code="mitra", + statement_type="challenges", theme_vectors={}, theme_id_to_info={}, + abusive_masked_at=[], is_discussion=True, + ) + assert pending is None + assert result["category_type"] == "Unknown/Unclear" + insert_mock.assert_awaited_once() + assert insert_mock.await_args.kwargs["category_type"] == "Unknown/Unclear" + asyncio.run(run_test()) + + +def test_theme_002_garbage_spam_statement_classified_unknown_unclear(monkeypatch): + async def run_test(): + conn = FakeConn() + monkeypatch.setattr(thematic_module, "insert_analysis_result", AsyncMock()) + result, pending = await thematic_module._run_local_classification( + conn=conn, statement="asdf asdf asdf asdf asdf", submission_id="1", tenant_code="mitra", + statement_type="challenges", theme_vectors={}, theme_id_to_info={}, + abusive_masked_at=[], is_discussion=True, + ) + assert pending is None + assert result["category_type"] == "Unknown/Unclear" + asyncio.run(run_test()) + + +def test_theme_003_pii_mask_tag_classified_flagged(monkeypatch): + async def run_test(): + conn = FakeConn() + insert_mock = AsyncMock() + monkeypatch.setattr(thematic_module, "insert_analysis_result", insert_mock) + result, pending = await thematic_module._run_local_classification( + conn=conn, statement="Met with at the village school today", + submission_id="1", tenant_code="mitra", statement_type="challenges", + theme_vectors={}, theme_id_to_info={}, abusive_masked_at=[], is_discussion=True, + ) + assert pending is None + assert result["category_type"] == "Flagged" + assert insert_mock.await_args.kwargs["category_type"] == "Flagged" + asyncio.run(run_test()) + + +def test_theme_004_abusive_flagged_column_classified_flagged(monkeypatch): + async def run_test(): + conn = FakeConn() + monkeypatch.setattr(thematic_module, "insert_analysis_result", AsyncMock()) + result, pending = await thematic_module._run_local_classification( + conn=conn, statement="This is a perfectly normal length statement here", + submission_id="1", tenant_code="mitra", statement_type="challenges", + theme_vectors={}, theme_id_to_info={}, abusive_masked_at=["challenges"], is_discussion=True, + ) + assert pending is None + assert result["category_type"] == "Flagged" + asyncio.run(run_test()) + + +def test_theme_005_local_similarity_match_classified_standard_no_llm(monkeypatch): + async def run_test(): + conn = FakeConn() + insert_mock = AsyncMock() + monkeypatch.setattr(thematic_module, "insert_analysis_result", insert_mock) + monkeypatch.setattr( + thematic_module, "get_theme_similarities", + MagicMock(return_value=[("theme-1", 0.9)]), + ) + settings_override(monkeypatch, thematic_module.settings, SIMILARITY_SCORE_THRESHOLD=0.65) + result, pending = await thematic_module._run_local_classification( + conn=conn, statement="A statement long enough to pass the word count gate", + submission_id="1", tenant_code="mitra", statement_type="challenges", + theme_vectors={"theme-1": "vec"}, theme_id_to_info={"theme-1": {"name": "Infra"}}, + abusive_masked_at=[], is_discussion=True, + ) + assert pending is None + assert result["category_type"] == "Standard" + assert result["theme_id"] == "theme-1" + assert result["confidence_score"] is None # no LLM used + asyncio.run(run_test()) + + +def test_theme_006_below_threshold_queued_for_llm_fallback(monkeypatch): + async def run_test(): + conn = FakeConn() + monkeypatch.setattr( + thematic_module, "get_theme_similarities", + MagicMock(return_value=[("theme-1", 0.1)]), + ) + settings_override(monkeypatch, thematic_module.settings, SIMILARITY_SCORE_THRESHOLD=0.65) + result, pending = await thematic_module._run_local_classification( + conn=conn, statement="A statement long enough to pass the word count gate", + submission_id="1", tenant_code="mitra", statement_type="challenges", + theme_vectors={"theme-1": "vec"}, theme_id_to_info={"theme-1": {"name": "Infra"}}, + abusive_masked_at=[], is_discussion=True, + ) + assert result is None + assert pending is not None + assert pending["diagnostics"]["llm_fallback"]["executed"] is True + asyncio.run(run_test()) + + +def test_theme_007_discussion_multi_theme_mapped_capped_at_max(monkeypatch): + is_discussion = True + resolved_items = [ + {"theme_id": f"t{i}", "confidence_score": 0.9 - i * 0.01, "justification": "j"} for i in range(5) + ] + settings_override(monkeypatch, thematic_module.settings, LLM_CONFIDENCE_SCORE_THRESHOLD=0.5) + qualifying = thematic_module._finalize_qualifying_themes(resolved_items, is_discussion) + assert len(qualifying) == thematic_module.MAX_MULTI_THEME_MATCHES + + +def test_theme_008_story_stays_single_theme_even_with_multiple_qualifying(): + resolved_items = [ + {"theme_id": "t1", "confidence_score": 0.9, "justification": "j"}, + {"theme_id": "t2", "confidence_score": 0.85, "justification": "j"}, + ] + qualifying = thematic_module._finalize_qualifying_themes(resolved_items, is_discussion=False) + assert len(qualifying) == 1 + + +def test_theme_009_llm_confidence_at_threshold_classified_standard(): + score = thematic_module._parse_confidence_score(0.8) + assert score == 0.8 + + +def test_theme_010_llm_confidence_below_threshold_would_resolve_others(monkeypatch): + settings_override(monkeypatch, thematic_module.settings, LLM_CONFIDENCE_SCORE_THRESHOLD=0.8) + resolved_items = [{"theme_id": "t1", "confidence_score": 0.5, "justification": "j"}] + qualifying = thematic_module._finalize_qualifying_themes(resolved_items, is_discussion=True) + assert qualifying == [] + + +def test_theme_011_non_numeric_confidence_score_rejected(): + assert thematic_module._parse_confidence_score("high") is None + assert thematic_module._parse_confidence_score(1.7) is None + assert thematic_module._parse_confidence_score(-0.1) is None + + +def test_theme_012_missing_statement_index_recovered_via_echoed_text(monkeypatch): + async def run_test(): + install_fake_llm(monkeypatch, content=json.dumps({ + "classified_data": [ + {"statement": "The exact echoed statement", "theme_name": "Infra", "confidence_score": 0.9, "justification": "j"} + ] + })) + conn = FakeConn() + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + install_fake_db(monkeypatch, thematic_module, conn) + monkeypatch.setattr(thematic_module, "insert_llm_log", AsyncMock()) + monkeypatch.setattr(thematic_module, "insert_analysis_result", AsyncMock()) + settings_override(monkeypatch, thematic_module.settings, LLM_CONFIDENCE_SCORE_THRESHOLD=0.5) + + results = await thematic_module._run_batched_llm_fallback( + pending_items=[{"statement": "The exact echoed statement", "statement_type": "challenges", + "is_discussion": True, "best_similarity": 0.1, "diagnostics": {"llm_fallback": {}}}], + approved_themes=[{"id": "theme-1", "name": "Infra"}], + theme_id_to_info={"theme-1": {"name": "Infra"}}, + submission_id="1", tenant_code="mitra", analysis_type="thematic_classification", + resolved_model="m", resolved_max_tokens=100, resolved_timeout=10, + ) + assert results[0]["category_type"] == "Standard" + asyncio.run(run_test()) + + +def test_theme_013_unmatchable_entry_dropped_statement_resolves_others(monkeypatch): + async def run_test(): + install_fake_llm(monkeypatch, content=json.dumps({ + "classified_data": [ + {"statement": "Something totally different", "theme_name": "Infra", "confidence_score": 0.9, "justification": "j"} + ] + })) + conn = FakeConn() + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + install_fake_db(monkeypatch, thematic_module, conn) + monkeypatch.setattr(thematic_module, "insert_llm_log", AsyncMock()) + insert_result_mock = AsyncMock() + monkeypatch.setattr(thematic_module, "insert_analysis_result", insert_result_mock) + + results = await thematic_module._run_batched_llm_fallback( + pending_items=[{"statement": "The real pending statement", "statement_type": "challenges", + "is_discussion": True, "best_similarity": 0.1, "diagnostics": {"llm_fallback": {}}}], + approved_themes=[{"id": "theme-1", "name": "Infra"}], + theme_id_to_info={"theme-1": {"name": "Infra"}}, + submission_id="1", tenant_code="mitra", analysis_type="thematic_classification", + resolved_model="m", resolved_max_tokens=100, resolved_timeout=10, + ) + assert results[0]["category_type"] == "Others" + assert insert_result_mock.await_args.kwargs["category_type"] == "Others" + asyncio.run(run_test()) + + +def test_theme_014_batched_llm_call_failure_raises_no_fabricated_result(monkeypatch): + async def run_test(): + install_failing_llm(monkeypatch, RuntimeError("network down")) + conn = FakeConn() + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + install_fake_db(monkeypatch, thematic_module, conn) + log_mock = AsyncMock() + monkeypatch.setattr(thematic_module, "insert_llm_log", log_mock) + insert_result_mock = AsyncMock() + monkeypatch.setattr(thematic_module, "insert_analysis_result", insert_result_mock) + + with pytest.raises(Exception): + await thematic_module._run_batched_llm_fallback( + pending_items=[{"statement": "x", "statement_type": "challenges", "is_discussion": True, + "best_similarity": 0.1, "diagnostics": {}}], + approved_themes=[], theme_id_to_info={}, + submission_id="1", tenant_code="mitra", analysis_type="thematic_classification", + resolved_model="m", resolved_max_tokens=100, resolved_timeout=10, + ) + insert_result_mock.assert_not_awaited() + assert log_mock.await_args.kwargs["status"] == "failed" + asyncio.run(run_test()) + + +def test_theme_015_json_parse_failure_raises_without_logging_content(monkeypatch, caplog): + async def run_test(): + marker = "SECRET_STATEMENT_TEXT" + install_fake_llm(monkeypatch, content=f"not json {marker}") + conn = FakeConn() + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + install_fake_db(monkeypatch, thematic_module, conn) + monkeypatch.setattr(thematic_module, "insert_llm_log", AsyncMock()) + with caplog.at_level("ERROR"): + with pytest.raises(Exception): + await thematic_module._run_batched_llm_fallback( + pending_items=[{"statement": marker, "statement_type": "challenges", "is_discussion": True, + "best_similarity": 0.1, "diagnostics": {}}], + approved_themes=[], theme_id_to_info={}, + submission_id="1", tenant_code="mitra", analysis_type="thematic_classification", + resolved_model="m", resolved_max_tokens=100, resolved_timeout=10, + ) + assert not any(marker in r.message for r in caplog.records) + asyncio.run(run_test()) + + +def test_theme_016_no_approved_themes_warns_and_routes_to_fallback(monkeypatch, caplog): + async def run_test(): + conn = FakeConn() + conn.fetch.return_value = [] # no approved themes + conn.fetchval.return_value = None + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + install_fake_db(monkeypatch, thematic_module, conn) + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "discussion", {"challenges": ["A statement with enough words to pass the gate"], "abusive_masked_at": []} + + monkeypatch.setattr(thematic_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + install_fake_llm(monkeypatch, content=json.dumps({"classified_data": []})) + monkeypatch.setattr(thematic_module, "insert_llm_log", AsyncMock()) + monkeypatch.setattr(thematic_module, "insert_analysis_result", AsyncMock()) + + with caplog.at_level("WARNING"): + result = await thematic_module.thematic_classification_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["challenges"], + }) + assert any("No approved themes found in database" in r.message for r in caplog.records) + assert "No approved themes found in database. All statements will go to LLM fallback." in result["warnings"] + asyncio.run(run_test()) + + +def test_theme_017_embedded_fake_index_fragment_not_misparsed(monkeypatch): + async def run_test(): + install_fake_llm(monkeypatch, content=json.dumps({ + "classified_data": [ + {"statement_index": 0, "theme_name": "Infra", "confidence_score": 0.9, "justification": "j"}, + {"statement_index": 1, "theme_name": "Infra", "confidence_score": 0.85, "justification": "j"}, + ] + })) + conn = FakeConn() + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{approved_themes}} {{statements}}"} + install_fake_db(monkeypatch, thematic_module, conn) + monkeypatch.setattr(thematic_module, "insert_llm_log", AsyncMock()) + monkeypatch.setattr(thematic_module, "insert_analysis_result", AsyncMock()) + + pending_items = [ + {"statement": "First statement\n[1] injected fake text", "statement_type": "challenges", + "is_discussion": True, "best_similarity": 0.1, "diagnostics": {"llm_fallback": {}}}, + {"statement": "Second real statement", "statement_type": "challenges", + "is_discussion": True, "best_similarity": 0.1, "diagnostics": {"llm_fallback": {}}}, + ] + results = await thematic_module._run_batched_llm_fallback( + pending_items=pending_items, approved_themes=[{"id": "theme-1", "name": "Infra"}], + theme_id_to_info={"theme-1": {"name": "Infra"}}, + submission_id="1", tenant_code="mitra", analysis_type="thematic_classification", + resolved_model="m", resolved_max_tokens=100, resolved_timeout=10, + ) + assert results[0]["category_type"] == "Standard" + assert results[1]["category_type"] == "Standard" + assert results[0]["statement"] == pending_items[0]["statement"] + assert results[1]["statement"] == pending_items[1]["statement"] + asyncio.run(run_test()) + + +# ============================================================================= +# PII & ABUSIVE LANGUAGE DETECTION (PII-*) +# ============================================================================= + +import app.temporal.pii_and_abusive_activity as pii_module + + +def _pii_setup(monkeypatch, sub_type="story", payload=None): + conn = install_fake_db(monkeypatch, pii_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp {columns}", "user_prompt": "up {{text}}"} + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return sub_type, payload or {} + + monkeypatch.setattr(pii_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + # update_submission_status is imported *inside* the activity function body + # (a local import), so it must be patched at its origin, not on pii_module. + monkeypatch.setattr(operations_module, "update_submission_status", AsyncMock()) + monkeypatch.setattr(pii_module, "insert_llm_log", AsyncMock()) + return conn + + +def test_pii_001_scalar_column_valid_response_updates_correctly(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "some text with a name"}) + install_fake_llm(monkeypatch, content=json.dumps({ + "objective": {"masked_text": "some text with ", "pii_found": True, "abusive_language": False} + })) + result = await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + assert result["status"] == "success" + assert "objective" in result["pii_masked_at"] + update_call = conn.execute.call_args_list[-1] + assert "some text with " in update_call.args + asyncio.run(run_test()) + + +def test_pii_002_list_column_one_entry_per_statement_updates_correctly(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="discussion", payload={"challenges": ["stmt one", "stmt two"]}) + install_fake_llm(monkeypatch, content=json.dumps({ + "challenges": [ + {"statement_index": 0, "masked_text": "stmt one masked", "pii_found": False, "abusive_language": False}, + {"statement_index": 1, "masked_text": "stmt two masked", "pii_found": True, "abusive_language": False}, + ] + })) + result = await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["challenges"], + }) + assert result["status"] == "success" + update_call = conn.execute.call_args_list[-1] + assert ["stmt one masked", "stmt two masked"] in update_call.args + asyncio.run(run_test()) + + +def test_pii_003_list_column_missing_statement_index_raises(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="discussion", payload={"challenges": ["stmt one", "stmt two"]}) + install_fake_llm(monkeypatch, content=json.dumps({ + "challenges": [ + {"statement_index": 0, "masked_text": "stmt one masked", "pii_found": False, "abusive_language": False}, + ] + })) + with pytest.raises(ValueError, match="missing masked entries for"): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["challenges"], + }) + asyncio.run(run_test()) + + +def test_pii_004_list_column_duplicate_statement_index_raises(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="discussion", payload={"challenges": ["stmt one", "stmt two"]}) + install_fake_llm(monkeypatch, content=json.dumps({ + "challenges": [ + {"statement_index": 0, "masked_text": "a", "pii_found": False, "abusive_language": False}, + {"statement_index": 0, "masked_text": "b", "pii_found": False, "abusive_language": False}, + ] + })) + with pytest.raises(ValueError, match="duplicate statement_index"): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["challenges"], + }) + asyncio.run(run_test()) + + +def test_pii_005_scalar_column_missing_masked_text_raises(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "text"}) + install_fake_llm(monkeypatch, content=json.dumps({"objective": {"pii_found": False, "abusive_language": False}})) + with pytest.raises(ValueError, match="missing 'masked_text'"): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + asyncio.run(run_test()) + + +def test_pii_006_scalar_column_wrong_shape_raises(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "text"}) + install_fake_llm(monkeypatch, content=json.dumps({"objective": "just a plain string, not an object"})) + with pytest.raises(ValueError, match="was not an object"): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + asyncio.run(run_test()) + + +def test_pii_007_pii_found_adds_column_to_pii_masked_at(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "text"}) + install_fake_llm(monkeypatch, content=json.dumps({ + "objective": {"masked_text": "masked", "pii_found": True, "abusive_language": False} + })) + result = await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + assert result["pii_masked_at"] == ["objective"] + asyncio.run(run_test()) + + +def test_pii_008_abusive_language_adds_column_to_abusive_masked_at(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "text"}) + install_fake_llm(monkeypatch, content=json.dumps({ + "objective": {"masked_text": "masked", "pii_found": False, "abusive_language": True} + })) + result = await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + assert result["abusive_masked_at"] == ["objective"] + asyncio.run(run_test()) + + +def test_pii_009_llm_failure_logged_status_failed(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "text"}) + install_failing_llm(monkeypatch, RuntimeError("transient network error")) + log_mock = AsyncMock() + monkeypatch.setattr(pii_module, "insert_llm_log", log_mock) + with pytest.raises(Exception): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + # insert_llm_log(conn, submission_id, tenant_code, model, analysis_type, + # prompt_version_id, prompt_tokens, completion_tokens, status, ...) + assert log_mock.await_args.args[8] == "failed" + asyncio.run(run_test()) + + +def test_pii_010_malformed_response_raises_without_logging_content(monkeypatch, caplog): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "text"}) + marker = "SUBMISSION_SECRET_TEXT" + install_fake_llm(monkeypatch, content=f"not json {marker}") + with caplog.at_level("ERROR"): + with pytest.raises(Exception): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + assert not any(marker in r.message for r in caplog.records) + asyncio.run(run_test()) + + +# ============================================================================= +# STORY RATING (RATING-*) +# ============================================================================= +# RATING-009 excluded: "no DB connection held during OpenRouter/PDF calls" is a +# live connection-pool-contention observation, not assertable via mocks. + +import app.temporal.story_rating_activity as rating_module + + +def test_rating_001_valid_pdf_download_produces_and_persists_rating(monkeypatch): + async def run_test(): + conn = install_fake_db(monkeypatch, rating_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{story_content}}"} + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "story", {"pdf_urls": ["https://example.com/x.pdf"], "challenge": None, "action_steps": None, "impact": None} + + monkeypatch.setattr(rating_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + monkeypatch.setattr(rating_module, "_fetch_story_content", MagicMock(return_value=("Extracted PDF text.", "pdf", 20))) + monkeypatch.setattr(rating_module, "insert_ranking_result", AsyncMock()) + monkeypatch.setattr(rating_module, "insert_llm_log", AsyncMock()) + + llm_result = { + "document_language": "en", "impact_and_outcome_score": 0.8, "impact_justification": "j", + "issue_and_challenge_score": 0.7, "issue_justification": "j", "action_steps_score": 0.6, + "action_justification": "j", "composite_score": 0.75, "tier": "Gold", "overall_summary": "s", + } + install_fake_llm(monkeypatch, content=json.dumps(llm_result)) + + result = await rating_module.story_rating_activity({"submission_id": "1", "tenant_code": "mitra"}) + assert result["status"] == "success" + assert result["tier"] == "Gold" + assert result["content_source"] == "pdf" + asyncio.run(run_test()) + + +def test_rating_002_pdf_failure_falls_back_to_fields(monkeypatch): + async def run_test(): + content, source, total_chars = rating_module._fetch_story_content( + pdf_url="https://example.com/broken.pdf", + challenge="A challenge statement here", action_steps="Some action steps", + impact="Some impact", submission_id="1", tenant_code="mitra", log_prefix="[test]", + ) + + def fake_download_that_fails(url, local_path): + raise RuntimeError("404 not found") + + monkeypatch.setattr(rating_module, "_download_file", MagicMock(side_effect=RuntimeError("404 not found"))) + content, source, total_chars = rating_module._fetch_story_content( + pdf_url="https://example.com/broken.pdf", + challenge="A challenge statement here", action_steps="Some action steps", + impact="Some impact", submission_id="1", tenant_code="mitra", log_prefix="[test]", + ) + assert source == "fields" + assert "A challenge statement here" in content + asyncio.run(run_test()) + + +def test_rating_003_no_pdf_no_fallback_fields_skips_gracefully(monkeypatch): + async def run_test(): + install_fake_db(monkeypatch, rating_module) + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "story", {"pdf_urls": [], "challenge": None, "action_steps": None, "impact": None} + + monkeypatch.setattr(rating_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + result = await rating_module.story_rating_activity({"submission_id": "1", "tenant_code": "mitra"}) + assert result["status"] == "skipped" + assert "no PDF content or fallback fields available" in result["reason"] + asyncio.run(run_test()) + + +def test_rating_004_llm_response_missing_required_field_raises(monkeypatch): + async def run_test(): + conn = install_fake_db(monkeypatch, rating_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{story_content}}"} + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "story", {"pdf_urls": [], "challenge": "some challenge text here", "action_steps": None, "impact": None} + + monkeypatch.setattr(rating_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + monkeypatch.setattr(rating_module, "insert_llm_log", AsyncMock()) + incomplete = {"document_language": "en", "composite_score": 0.5, "tier": "Silver", "overall_summary": "s"} + install_fake_llm(monkeypatch, content=json.dumps(incomplete)) + + with pytest.raises(ValueError, match="missing required fields"): + await rating_module.story_rating_activity({"submission_id": "1", "tenant_code": "mitra"}) + asyncio.run(run_test()) + + +def test_rating_005_llm_response_score_out_of_range_raises(monkeypatch): + async def run_test(): + conn = install_fake_db(monkeypatch, rating_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{story_content}}"} + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "story", {"pdf_urls": [], "challenge": "some challenge text here", "action_steps": None, "impact": None} + + monkeypatch.setattr(rating_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + monkeypatch.setattr(rating_module, "insert_llm_log", AsyncMock()) + bad = { + "document_language": "en", "impact_and_outcome_score": 1.4, "impact_justification": "j", + "issue_and_challenge_score": 0.7, "issue_justification": "j", "action_steps_score": 0.6, + "action_justification": "j", "composite_score": 0.75, "tier": "Gold", "overall_summary": "s", + } + install_fake_llm(monkeypatch, content=json.dumps(bad)) + + with pytest.raises(ValueError, match="outside the valid 0.0-1.0 range"): + await rating_module.story_rating_activity({"submission_id": "1", "tenant_code": "mitra"}) + asyncio.run(run_test()) + + +def test_rating_006_persistence_failure_mid_sequence_rolls_back(monkeypatch): + """FakeConn.transaction() returns the connection itself as an async context + manager (no real rollback semantics) — this asserts the code *attempts* the + delete+insert+log inside one `async with conn.transaction():` block, which + is what makes a real Postgres rollback possible; it can't itself prove + Postgres rolled back without a real DB.""" + async def run_test(): + conn = install_fake_db(monkeypatch, rating_module) + conn.fetchrow.return_value = {"id": "pv-1", "system_prompt": "sp", "user_prompt": "up {{story_content}}"} + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "story", {"pdf_urls": [], "challenge": "some challenge text here", "action_steps": None, "impact": None} + + monkeypatch.setattr(rating_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + monkeypatch.setattr(rating_module, "insert_ranking_result", AsyncMock()) + monkeypatch.setattr(rating_module, "insert_llm_log", AsyncMock(side_effect=RuntimeError("log insert failed"))) + good = { + "document_language": "en", "impact_and_outcome_score": 0.8, "impact_justification": "j", + "issue_and_challenge_score": 0.7, "issue_justification": "j", "action_steps_score": 0.6, + "action_justification": "j", "composite_score": 0.75, "tier": "Gold", "overall_summary": "s", + } + install_fake_llm(monkeypatch, content=json.dumps(good)) + + with pytest.raises(RuntimeError, match="log insert failed"): + await rating_module.story_rating_activity({"submission_id": "1", "tenant_code": "mitra"}) + # transaction() was entered exactly once, wrapping both writes + conn.transaction.assert_called_once() + asyncio.run(run_test()) + + +def test_rating_007_non_story_submission_skipped(monkeypatch): + async def run_test(): + install_fake_db(monkeypatch, rating_module) + + async def fake_get_submission_type_and_payload(c, sid, tenant): + return "discussion", {} + + monkeypatch.setattr(rating_module, "get_submission_type_and_payload", fake_get_submission_type_and_payload) + result = await rating_module.story_rating_activity({"submission_id": "1", "tenant_code": "mitra"}) + assert result["status"] == "skipped" + assert "story_rating only applies to story submissions" in result["reason"] + asyncio.run(run_test()) + + +def test_rating_008_relative_pdf_url_without_media_base_url_raises(monkeypatch): + settings_override(monkeypatch, rating_module.settings, MEDIA_BASE_URL="") + with pytest.raises(ValueError, match="MEDIA_BASE_URL is not configured"): + rating_module._resolve_url("relative/path/to/file.pdf") + + +# ============================================================================= +# BATCH PROCESSING WORKFLOW (BATCH-*) +# ============================================================================= +# BATCH-006 excluded: the SKIP overlap policy is enforced by Temporal's own +# server-side scheduler, not application code — nothing here to unit test. + +import app.temporal.workflows as workflows_module + + +def test_batch_001_small_queue_drains_in_single_chunk(monkeypatch): + async def run_test(): + pending = [{"submission_id": "1", "tenant_code": "mitra", "submission_type": "story", "process_steps": []}] + exec_mock, _, _ = install_fake_workflow_context( + monkeypatch, + activity_results={workflows_module.fetch_pending_submissions_activity: pending}, + child_workflow_results={workflows_module.ConfigDrivenProcessingWorkflow.run: {"status": "success"}}, + ) + # second fetch call (after the chunk) must return [] to end the loop + call_count = {"n": 0} + + async def fake_execute_activity(activity_fn, *args, **kwargs): + if activity_fn is workflows_module.fetch_pending_submissions_activity: + call_count["n"] += 1 + return pending if call_count["n"] == 1 else [] + return None + + exec_mock.side_effect = fake_execute_activity + + wf = workflows_module.BatchProcessingWorkflow() + result = await wf.run(batch_size=100) + assert result["processed_count"] == 1 + assert result["success_count"] == 1 + assert result["chunks"] == 1 + asyncio.run(run_test()) + + +def test_batch_002_large_queue_fans_out_across_multiple_chunks(monkeypatch): + async def run_test(): + chunk1 = [{"submission_id": str(i), "tenant_code": "mitra", "submission_type": "story", "process_steps": []} for i in range(2)] + chunk2 = [{"submission_id": str(i), "tenant_code": "mitra", "submission_type": "story", "process_steps": []} for i in range(2, 3)] + calls = {"n": 0} + + exec_mock, _, _ = install_fake_workflow_context(monkeypatch) + + async def fake_execute_activity(activity_fn, *args, **kwargs): + if activity_fn is workflows_module.fetch_pending_submissions_activity: + calls["n"] += 1 + if calls["n"] == 1: + return chunk1 + elif calls["n"] == 2: + return chunk2 + return [] + return None + + exec_mock.side_effect = fake_execute_activity + + wf = workflows_module.BatchProcessingWorkflow() + result = await wf.run(batch_size=2) + assert result["processed_count"] == 3 + assert result["chunks"] == 2 + asyncio.run(run_test()) + + +def test_batch_003_no_pending_submissions_returns_zero(monkeypatch): + async def run_test(): + install_fake_workflow_context( + monkeypatch, activity_results={workflows_module.fetch_pending_submissions_activity: []}, + ) + wf = workflows_module.BatchProcessingWorkflow() + result = await wf.run(batch_size=100) + assert result == {"processed_count": 0, "message": "No pending submissions found."} + asyncio.run(run_test()) + + +def test_batch_004_one_child_failure_counted_without_halting_rest(monkeypatch): + async def run_test(): + pending = [ + {"submission_id": "1", "tenant_code": "mitra", "submission_type": "story", "process_steps": []}, + {"submission_id": "2", "tenant_code": "mitra", "submission_type": "story", "process_steps": []}, + ] + calls = {"n": 0} + exec_mock, _, _ = install_fake_workflow_context(monkeypatch) + + async def fake_execute_activity(activity_fn, *args, **kwargs): + if activity_fn is workflows_module.fetch_pending_submissions_activity: + calls["n"] += 1 + return pending if calls["n"] == 1 else [] + return None + + exec_mock.side_effect = fake_execute_activity + + async def fake_execute_child_workflow(run_fn, payload, **kwargs): + if payload["submission_id"] == "1": + raise RuntimeError("child failed") + return {"status": "success"} + + monkeypatch.setattr(workflows_module.workflow, "execute_child_workflow", fake_execute_child_workflow) + + wf = workflows_module.BatchProcessingWorkflow() + result = await wf.run(batch_size=100) + assert result["failed_count"] == 1 + assert result["success_count"] == 1 + asyncio.run(run_test()) + + +def test_batch_005_exceeding_max_per_run_triggers_continue_as_new(monkeypatch): + async def run_test(): + big_chunk = [ + {"submission_id": str(i), "tenant_code": "mitra", "submission_type": "story", "process_steps": []} + for i in range(workflows_module.MAX_SUBMISSIONS_PER_RUN) + ] + exec_mock, continue_as_new_calls, ContinueAsNew = install_fake_workflow_context( + monkeypatch, + activity_results={workflows_module.fetch_pending_submissions_activity: big_chunk}, + child_workflow_results={workflows_module.ConfigDrivenProcessingWorkflow.run: {"status": "success"}}, + ) + monkeypatch.setattr( + workflows_module.workflow, "execute_child_workflow", + AsyncMock(return_value={"status": "success"}), + ) + + wf = workflows_module.BatchProcessingWorkflow() + with pytest.raises(ContinueAsNew): + await wf.run(batch_size=workflows_module.MAX_SUBMISSIONS_PER_RUN) + + assert len(continue_as_new_calls) == 1 + carried_batch_size, carry_over = continue_as_new_calls[0] + assert carry_over["total_processed"] == workflows_module.MAX_SUBMISSIONS_PER_RUN + asyncio.run(run_test()) + + +# ============================================================================= +# REAL-TIME / MODE HANDLING (MODE-*) +# ============================================================================= + +def test_mode_001_real_time_triggers_workflow_immediately(monkeypatch): + async def run_test(): + consumer, insert_mock, _, trigger_mock = _consumer_with_mocks(monkeypatch) + settings_override(monkeypatch, consumer_module.settings, PROCESSING_MODE="real-time") + event = _fixture("create", "create_discussion.json") + await consumer.process_message(json.dumps(event)) + trigger_mock.assert_awaited_once() + asyncio.run(run_test()) + + +def test_mode_002_batch_mode_leaves_submission_pending(monkeypatch): + async def run_test(): + consumer, insert_mock, _, trigger_mock = _consumer_with_mocks(monkeypatch) + settings_override(monkeypatch, consumer_module.settings, PROCESSING_MODE="batch") + event = _fixture("create", "create_discussion.json") + await consumer.process_message(json.dumps(event)) + insert_mock.assert_awaited_once() + trigger_mock.assert_not_awaited() + asyncio.run(run_test()) + + +def test_mode_003_temporal_unreachable_heals_connection_on_next_message(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + consumer.temporal_client = None + install_fake_db(monkeypatch, consumer_module) + settings_override(monkeypatch, consumer_module.settings, PROCESSING_MODE="real-time") + mock_update_status = AsyncMock() + monkeypatch.setattr(consumer_module, "update_submission_status", mock_update_status) + + mock_client = MagicMock() + mock_client.start_workflow = AsyncMock() + monkeypatch.setattr(consumer_module.Client, "connect", AsyncMock(return_value=mock_client)) + + await consumer._trigger_realtime_workflow("sub1", "tenant1", "story") + + mock_client.start_workflow.assert_awaited_once() + mock_update_status.assert_awaited_once_with(ANY, "sub1", "tenant1", "processing") + assert consumer.temporal_client is mock_client + asyncio.run(run_test()) + + +def test_mode_003b_temporal_connect_failure_leaves_client_none(monkeypatch): + async def run_test(): + consumer = consumer_module.IngestionConsumer() + consumer.temporal_client = None + monkeypatch.setattr(consumer_module.Client, "connect", AsyncMock(side_effect=Exception("unreachable"))) + await consumer._trigger_realtime_workflow("sub1", "tenant1", "story") + assert consumer.temporal_client is None + asyncio.run(run_test()) + + +# ============================================================================= +# LLM & COST TRACKING (LLM-*) +# ============================================================================= + +from app.services.llm import openrouter_chat_completion, split_llm_usage + + +def test_llm_001_returns_real_usage_not_estimate(monkeypatch): + settings_override(monkeypatch, settings, OPENROUTER_API_KEY="test-key") + install_fake_llm(monkeypatch, content="Hello world", usage={"prompt_tokens": 42, "completion_tokens": 13, "total_tokens": 55, "cost": 0.002}) + content, usage = openrouter_chat_completion("a prompt") + assert content == "Hello world" + assert usage["prompt_tokens"] == 42 + assert usage["cost"] == 0.002 + + +def test_llm_002_split_llm_usage_separates_tokens_from_metadata(): + prompt_tokens, completion_tokens, meta = split_llm_usage( + {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15, "cost": 0.001} + ) + assert (prompt_tokens, completion_tokens) == (10, 5) + assert meta == {"cost": 0.001} + + +def test_llm_003_fallback_estimate_only_when_no_usage_ever_obtained(monkeypatch): + async def run_test(): + conn = _pii_setup(monkeypatch, sub_type="story", payload={"objective": "a b c d e f g h"}) + install_failing_llm(monkeypatch, RuntimeError("invalid api key")) + log_mock = AsyncMock() + monkeypatch.setattr(pii_module, "insert_llm_log", log_mock) + with pytest.raises(Exception): + await pii_module.pii_and_abusive_language_detection_activity({ + "submission_id": "1", "tenant_code": "mitra", "target_columns": ["objective"], + }) + kwargs = log_mock.await_args.kwargs + assert kwargs["meta_data"] is None + assert log_mock.await_args.args[6] > 0 # prompt_tokens: word-count estimate, not zero + asyncio.run(run_test()) + + +# ============================================================================= +# CSV UPLOAD & PROCESS API (UPLOAD-*) +# ============================================================================= + +from app.api.router import api_router +from app.api.exceptions import register_exception_handlers +import app.database.operations as operations_module +import app.api.services.uploads as uploads_service_module + + +def _build_test_app() -> FastAPI: + app = FastAPI() + register_exception_handlers(app) + app.include_router(api_router) + return app + + +def _auth_headers(): + return {"Authorization": f"Bearer {settings.AUTH_TOKEN}"} + + +def _csv_file(name: str): + path = CSV_FIXTURE_ROOT / name + return {"file": (name, path.read_bytes(), "text/csv")} + + +@pytest.fixture +def test_client(): + # raise_server_exceptions=False so an unhandled exception in the route + # (e.g. RuntimeError bubbling out of handle_upload) comes back as the real + # HTTP 500 response a live client would see, instead of re-raising in-process. + return TestClient(_build_test_app(), raise_server_exceptions=False) + + +def test_upload_001_missing_auth_header_rejected(test_client): + resp = test_client.post("/v1/upload/") + assert resp.status_code == 403 + + +def test_upload_002_invalid_bearer_token_rejected(test_client): + resp = test_client.post("/v1/upload/", headers={"Authorization": "Bearer wrong-token"}) + assert resp.status_code == 401 + + +def test_upload_003_valid_story_csv_uploads_successfully(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + monkeypatch.setattr(operations_module, "insert_upload_record", AsyncMock(return_value=1)) + gcs_client, _ = make_fake_gcs_client() + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(return_value="path/to/file.csv")) + settings_override(monkeypatch, settings, PROCESSING_MODE="batch") # skip the Temporal trigger + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "pending" + + +def test_upload_004_valid_discussion_csv_uploads_successfully(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + monkeypatch.setattr(operations_module, "insert_upload_record", AsyncMock(return_value=2)) + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(return_value="path/to/file.csv")) + settings_override(monkeypatch, settings, PROCESSING_MODE="batch") + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "discussion", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_discussion.csv"), + ) + assert resp.status_code == 200 + assert resp.json()["report_type"] if False else resp.json()["status"] == "pending" + + +def test_upload_005_invalid_report_type_rejected(test_client): + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "survey", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 400 + + +def test_upload_006_non_csv_extension_rejected(test_client): + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files={"file": ("not_a_csv.txt", b"id,Title\n1,x\n", "text/plain")}, + ) + assert resp.status_code == 400 + + +def test_upload_007_empty_file_rejected(test_client): + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files={"file": ("empty.csv", b"", "text/csv")}, + ) + assert resp.status_code == 400 + + +def test_upload_008_oversized_file_rejected(test_client, monkeypatch): + settings_override(monkeypatch, settings, MAX_CSV_UPLOAD_BYTES=10) + big_content = b"id,Title\n" + b"1,x\n" * 100 + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files={"file": ("big.csv", big_content, "text/csv")}, + ) + assert resp.status_code == 413 + + +def test_upload_009_missing_columns_rejected_no_side_effects(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + insert_mock = AsyncMock() + upload_mock = MagicMock() + monkeypatch.setattr(operations_module, "insert_upload_record", insert_mock) + monkeypatch.setattr(uploads_service_module, "upload_csv", upload_mock) + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("missing_columns.csv"), + ) + assert resp.status_code == 400 + assert "Missing columns" in str(resp.json()["errors"]) + insert_mock.assert_not_awaited() + upload_mock.assert_not_called() + + +def test_upload_010_extra_columns_rejected_no_side_effects(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + insert_mock = AsyncMock() + upload_mock = MagicMock() + monkeypatch.setattr(operations_module, "insert_upload_record", insert_mock) + monkeypatch.setattr(uploads_service_module, "upload_csv", upload_mock) + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("extra_columns.csv"), + ) + assert resp.status_code == 400 + assert "Extra/unexpected columns" in str(resp.json()["errors"]) + insert_mock.assert_not_awaited() + upload_mock.assert_not_called() + + +def test_upload_011_malformed_csv_content_rejected(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("malformed.csv"), + ) + assert resp.status_code == 400 + + +def test_upload_012_duplicate_file_rejected(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=True)) + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 400 + assert "FILE ALREADY EXISTS" in resp.json()["detail"] + + +def test_upload_013_tenant_code_defaults_to_mitra(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + captured = {} + + async def fake_insert_upload_record(**kwargs): + captured.update(kwargs) + return 1 + + monkeypatch.setattr(operations_module, "insert_upload_record", fake_insert_upload_record) + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(return_value="path")) + settings_override(monkeypatch, settings, PROCESSING_MODE="batch") + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L"}, # no tenant_code + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 200 + assert captured["meta_data"]["tenant_code"] == "mitra" + + +def test_upload_014_real_time_mode_triggers_workflow_immediately(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + monkeypatch.setattr(operations_module, "insert_upload_record", AsyncMock(return_value=1)) + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(return_value="path")) + settings_override(monkeypatch, settings, PROCESSING_MODE="real-time") + + start_workflow_mock = AsyncMock() + mock_client = MagicMock() + mock_client.start_workflow = start_workflow_mock + monkeypatch.setattr(uploads_service_module.Client, "connect", AsyncMock(return_value=mock_client)) + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 200 + start_workflow_mock.assert_awaited_once() + + +def test_upload_015_batch_mode_leaves_upload_pending_no_workflow(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + monkeypatch.setattr(operations_module, "insert_upload_record", AsyncMock(return_value=1)) + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(return_value="path")) + settings_override(monkeypatch, settings, PROCESSING_MODE="batch") + connect_mock = AsyncMock() + monkeypatch.setattr(uploads_service_module.Client, "connect", connect_mock) + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "pending" + connect_mock.assert_not_awaited() + + +def test_upload_016_gcs_upload_failure_prevents_db_row(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + insert_mock = AsyncMock() + monkeypatch.setattr(operations_module, "insert_upload_record", insert_mock) + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(side_effect=RuntimeError("bucket unreachable"))) + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 500 + insert_mock.assert_not_awaited() + + +def test_upload_017_temporal_unreachable_marks_on_hold(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "check_duplicate_file", AsyncMock(return_value=False)) + monkeypatch.setattr(operations_module, "insert_upload_record", AsyncMock(return_value=1)) + monkeypatch.setattr(uploads_service_module, "upload_csv", MagicMock(return_value="path")) + settings_override(monkeypatch, settings, PROCESSING_MODE="real-time") + monkeypatch.setattr(uploads_service_module.Client, "connect", AsyncMock(side_effect=Exception("temporal down"))) + update_status_mock = AsyncMock() + monkeypatch.setattr(operations_module, "update_status", update_status_mock) + + resp = test_client.post( + "/v1/upload/", headers=_auth_headers(), + data={"report_type": "story", "program_name": "P", "leader_category": "L", "tenant_code": "mitra"}, + files=_csv_file("valid_story.csv"), + ) + assert resp.status_code == 500 + update_status_mock.assert_awaited_once() + assert update_status_mock.await_args.args[1] == "on_hold" + + +def test_upload_018_process_pending_record_starts_workflow(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "get_record", AsyncMock(return_value={"status": "pending"})) + monkeypatch.setattr(operations_module, "try_claim_for_processing", AsyncMock(return_value="success")) + start_workflow_mock = AsyncMock() + mock_client = MagicMock() + mock_client.start_workflow = start_workflow_mock + monkeypatch.setattr(uploads_service_module.Client, "connect", AsyncMock(return_value=mock_client)) + + resp = test_client.post("/v1/process/csv/1", headers=_auth_headers()) + assert resp.status_code == 200 + assert resp.json()["status"] == "success" + start_workflow_mock.assert_awaited_once() + + +def test_upload_019_process_nonexistent_record_404(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "get_record", AsyncMock(return_value=None)) + resp = test_client.post("/v1/process/csv/999999", headers=_auth_headers()) + assert resp.status_code == 404 + + +def test_upload_020_process_already_in_progress_409(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "get_record", AsyncMock(return_value={"status": "in_progress"})) + resp = test_client.post("/v1/process/csv/1", headers=_auth_headers()) + assert resp.status_code == 409 + + +def test_upload_021_reprocessing_terminal_status_409(test_client, monkeypatch): + monkeypatch.setattr(operations_module, "get_record", AsyncMock(return_value={"status": "success"})) + resp = test_client.post("/v1/process/csv/1", headers=_auth_headers()) + assert resp.status_code == 409 + assert "Only pending records can be processed" in resp.json()["detail"] + + +def test_upload_022_process_endpoint_requires_auth(test_client): + resp = test_client.post("/v1/process/csv/1") + assert resp.status_code == 403 + + +def test_upload_023_concurrent_process_calls_race_safe(): + """try_claim_for_processing's UPDATE ... WHERE status != 'in_progress' RETURNING + status is an atomic single-statement compare-and-swap at the SQL level — with + no real Postgres connection, this test documents/asserts the *call shape* + (single UPDATE...RETURNING statement, not a separate SELECT-then-UPDATE) that + makes it race-safe, rather than proving concurrency safety itself.""" + import inspect + from app.database import operations + source = inspect.getsource(operations.try_claim_for_processing) + assert "WHERE id = $1 AND status != 'in_progress'" in source + assert "RETURNING status" in source + + +def test_upload_024_missing_session_id_skipped_prepublish(monkeypatch): + async def run_test(): + import app.temporal.csv_processing_activity as csv_activity_module + conn = install_fake_db(monkeypatch, csv_activity_module) + conn.fetchrow.return_value = None # no programs/leader_category match -> UUID fallback path + record = { + "id": 1, "report_type": "story", "cloud_storage_path": "path/to/file.csv", + "leader_category": "L", "program_name": "P", "meta_data": {"tenant_code": "mitra"}, + } + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "get_record", AsyncMock(return_value=record)) + update_status_mock = AsyncMock() + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "update_status", update_status_mock) + monkeypatch.setattr(csv_activity_module, "fetch_csv", MagicMock(return_value=b"raw")) + + import pandas as pd + df = pd.DataFrame([{"id": "5004", "Title": "t", "Session ID": ""}]) + monkeypatch.setattr(csv_activity_module, "load_csv", MagicMock(return_value=df)) + monkeypatch.setattr(csv_activity_module, "validate_columns", MagicMock(return_value=(True, []))) + + push_mock = MagicMock() + monkeypatch.setattr(csv_activity_module, "_push_rows_sync", push_mock) + + result = await csv_activity_module.csv_push_to_kafka_activity(1) + assert result["rows_pushed"] == 0 + assert len(result["schema_validation_errors"]) == 1 + assert "'sessionId' is empty" in result["schema_validation_errors"][0]["problems"] + push_mock.assert_not_called() + asyncio.run(run_test()) + + +def test_upload_025_missing_other_required_fields_skipped_and_recorded(monkeypatch): + async def run_test(): + import app.temporal.csv_processing_activity as csv_activity_module + conn = install_fake_db(monkeypatch, csv_activity_module) + conn.fetchrow.return_value = None + record = { + "id": 1, "report_type": "story", "cloud_storage_path": "path/to/file.csv", + "leader_category": "L", "program_name": "P", "meta_data": {"tenant_code": "mitra"}, + } + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "get_record", AsyncMock(return_value=record)) + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "update_status", AsyncMock()) + monkeypatch.setattr(csv_activity_module, "fetch_csv", MagicMock(return_value=b"raw")) + + import pandas as pd + df = pd.DataFrame([{"id": "5004", "Title": "t", "Session ID": "sess-1"}]) + monkeypatch.setattr(csv_activity_module, "load_csv", MagicMock(return_value=df)) + monkeypatch.setattr(csv_activity_module, "validate_columns", MagicMock(return_value=(True, []))) + monkeypatch.setattr(csv_activity_module, "_push_rows_sync", MagicMock()) + + result = await csv_activity_module.csv_push_to_kafka_activity(1) + problems = result["schema_validation_errors"][0]["problems"] + assert any("transcriptLink" in p for p in problems) + asyncio.run(run_test()) + + +def test_upload_026_complete_row_still_fails_on_pdf_urls_masked(monkeypatch): + async def run_test(): + import app.temporal.csv_processing_activity as csv_activity_module + conn = install_fake_db(monkeypatch, csv_activity_module) + conn.fetchrow.return_value = None + record = { + "id": 1, "report_type": "story", "cloud_storage_path": "path/to/file.csv", + "leader_category": "L", "program_name": "P", "meta_data": {"tenant_code": "mitra"}, + } + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "get_record", AsyncMock(return_value=record)) + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "update_status", AsyncMock()) + monkeypatch.setattr(csv_activity_module, "fetch_csv", MagicMock(return_value=b"raw")) + + import pandas as pd + df = pd.DataFrame([{ + "id": "5001", "Title": "t", "Session ID": "sess-1", "Objective": "obj text here", + "Challenges": "a challenge", "Action Steps": "a step", "Impact": "some impact", + "Transcript Link": "https://example.com/t", "Blurb": "a blurb", "Content": "content here", + "Pdf": "https://example.com/x.pdf", + "District": "Patna", "Organization": "Org", "Location": "Patna, Bihar", + "Duration": "30 minutes", + }]) + monkeypatch.setattr(csv_activity_module, "load_csv", MagicMock(return_value=df)) + monkeypatch.setattr(csv_activity_module, "validate_columns", MagicMock(return_value=(True, []))) + monkeypatch.setattr(csv_activity_module, "_push_rows_sync", MagicMock()) + + result = await csv_activity_module.csv_push_to_kafka_activity(1) + assert result["rows_pushed"] == 0 + assert result["schema_validation_errors"][0]["problems"] == ["'data.pdfUrls.masked' is missing"] + asyncio.run(run_test()) + + +def test_upload_027_kafka_unreachable_marks_on_hold(monkeypatch): + async def run_test(): + import app.temporal.csv_processing_activity as csv_activity_module + conn = install_fake_db(monkeypatch, csv_activity_module) + conn.fetchrow.return_value = None + record = { + "id": 1, "report_type": "discussion", "cloud_storage_path": "path/to/file.csv", + "leader_category": "L", "program_name": "P", "meta_data": {"tenant_code": "mitra"}, + } + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "get_record", AsyncMock(return_value=record)) + update_status_mock = AsyncMock() + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "update_status", update_status_mock) + monkeypatch.setattr(csv_activity_module, "fetch_csv", MagicMock(return_value=b"raw")) + + import pandas as pd + df = pd.DataFrame([{ + "id": "6001", "Title": "t", "Session ID": "sess-1", "Challenges": "a challenge", + "Solutions": "a solution", "Transcript Link": "https://example.com/t", + "PDF Urls": "https://example.com/x.pdf", + }]) + monkeypatch.setattr(csv_activity_module, "load_csv", MagicMock(return_value=df)) + monkeypatch.setattr(csv_activity_module, "validate_columns", MagicMock(return_value=(True, []))) + monkeypatch.setattr( + csv_activity_module, "validate_ingestion_schema", + MagicMock(return_value=[]), # pretend it passes, to reach the Kafka push + ) + monkeypatch.setattr(csv_activity_module, "_push_rows_sync", MagicMock(side_effect=RuntimeError("broker down"))) + + with pytest.raises(RuntimeError, match="broker down"): + await csv_activity_module.csv_push_to_kafka_activity(1) + assert update_status_mock.await_args.args[1] == "on_hold" + asyncio.run(run_test()) + + +def test_upload_028_missing_program_leader_match_falls_back_to_uuid(monkeypatch): + async def run_test(): + import app.temporal.csv_processing_activity as csv_activity_module + import uuid as uuid_module + conn = install_fake_db(monkeypatch, csv_activity_module) + conn.fetchrow.return_value = None # no leader_category/programs match + record = { + "id": 1, "report_type": "story", "cloud_storage_path": "path/to/file.csv", + "leader_category": "Never Seen Before Leader", "program_name": "Never Seen Before Program", + "meta_data": {"tenant_code": "mitra"}, + } + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "get_record", AsyncMock(return_value=record)) + monkeypatch.setattr(csv_activity_module.csv_upload_repo, "update_status", AsyncMock()) + monkeypatch.setattr(csv_activity_module, "fetch_csv", MagicMock(return_value=b"raw")) + + import pandas as pd + df = pd.DataFrame([{"id": "5001", "Title": "t", "Session ID": "sess-1"}]) + monkeypatch.setattr(csv_activity_module, "load_csv", MagicMock(return_value=df)) + monkeypatch.setattr(csv_activity_module, "validate_columns", MagicMock(return_value=(True, []))) + + captured_payloads = [] + + def fake_push(payloads): + captured_payloads.extend(payloads) + + monkeypatch.setattr(csv_activity_module, "_push_rows_sync", fake_push) + monkeypatch.setattr(csv_activity_module, "validate_ingestion_schema", MagicMock(return_value=[])) + + await csv_activity_module.csv_push_to_kafka_activity(1) + assert len(captured_payloads) == 1 + payload = json.loads(captured_payloads[0][0]) + assert uuid_module.UUID(payload["tags"]["leaderCategoryId"]) # a real generated UUID, not a DB id + assert uuid_module.UUID(payload["tags"]["programId"]) + asyncio.run(run_test()) + + +def test_upload_029_batch_workflow_fans_out_pending_csv_uploads(monkeypatch): + async def run_test(): + exec_mock, _, _ = install_fake_workflow_context( + monkeypatch, + activity_results={workflows_module.fetch_pending_csv_uploads_activity: [1, 2]}, + ) + child_calls = [] + + async def fake_execute_child_workflow(run_fn, record_id, **kwargs): + child_calls.append(record_id) + return {"status": "success"} + + monkeypatch.setattr(workflows_module.workflow, "execute_child_workflow", fake_execute_child_workflow) + + wf = workflows_module.CsvBatchProcessingWorkflow() + result = await wf.run() + assert result["processed_count"] == 2 + assert sorted(child_calls) == [1, 2] + asyncio.run(run_test()) + + +def test_upload_030_batch_workflow_empty_queue_returns_zero(monkeypatch): + async def run_test(): + install_fake_workflow_context( + monkeypatch, activity_results={workflows_module.fetch_pending_csv_uploads_activity: []}, + ) + wf = workflows_module.CsvBatchProcessingWorkflow() + result = await wf.run() + assert result == {"processed_count": 0, "message": "No pending CSV uploads found."} + asyncio.run(run_test()) + + +def test_upload_031_csv_batch_schedule_registers_in_batch_mode(monkeypatch): + async def run_test(): + import app.temporal.worker as worker_module + settings_override(monkeypatch, worker_module.settings, PROCESSING_MODE="batch") + monkeypatch.setattr(worker_module.db, "connect", AsyncMock()) + monkeypatch.setattr(worker_module.db, "disconnect", AsyncMock()) + monkeypatch.setattr(worker_module, "Worker", MagicMock(return_value=MagicMock(run=AsyncMock()))) + + mock_client = MagicMock() + create_schedule_mock = AsyncMock() + mock_client.create_schedule = create_schedule_mock + monkeypatch.setattr(worker_module.Client, "connect", AsyncMock(return_value=mock_client)) + + await worker_module.start_worker() + + schedule_ids = [c.kwargs.get("id") for c in create_schedule_mock.await_args_list] + assert "csv-batch-processing" in schedule_ids + assert "daily-batch-processing" in schedule_ids + asyncio.run(run_test()) + + +def test_upload_032_stale_schedules_deleted_in_realtime_mode(monkeypatch): + async def run_test(): + import app.temporal.worker as worker_module + settings_override(monkeypatch, worker_module.settings, PROCESSING_MODE="real-time") + monkeypatch.setattr(worker_module.db, "connect", AsyncMock()) + monkeypatch.setattr(worker_module.db, "disconnect", AsyncMock()) + monkeypatch.setattr(worker_module, "Worker", MagicMock(return_value=MagicMock(run=AsyncMock()))) + + mock_client = MagicMock() + deleted_schedules = [] + + def fake_get_schedule_handle(sched_id): + handle = MagicMock() + + async def fake_delete(): + deleted_schedules.append(sched_id) + + handle.delete = fake_delete + return handle + + mock_client.get_schedule_handle = MagicMock(side_effect=fake_get_schedule_handle) + monkeypatch.setattr(worker_module.Client, "connect", AsyncMock(return_value=mock_client)) + + await worker_module.start_worker() + + assert set(deleted_schedules) == {"csv-batch-processing", "daily-batch-processing"} + asyncio.run(run_test()) From 61f16175eda45a6d7ee5d2039d79e40a21019d1e Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:25:30 +0530 Subject: [PATCH 02/10] added the supporting files --- tests/csv_uploads/empty.csv | 0 tests/csv_uploads/extra_columns.csv | 2 ++ tests/csv_uploads/malformed.csv | 2 ++ tests/csv_uploads/missing_columns.csv | 2 ++ .../csv_uploads/missing_session_id_value.csv | 2 ++ tests/csv_uploads/not_a_csv.txt | 2 ++ tests/csv_uploads/valid_discussion.csv | 2 ++ tests/csv_uploads/valid_story.csv | 2 ++ ...eate_discussion_multi_statement_array.json | 36 +++++++++++++++++++ .../create_discussion_multi_theme_llm.json | 34 ++++++++++++++++++ .../create_discussion_multi_theme_local.json | 34 ++++++++++++++++++ ...eate_story_multi_barrier_single_theme.json | 34 ++++++++++++++++++ 12 files changed, 152 insertions(+) create mode 100644 tests/csv_uploads/empty.csv create mode 100644 tests/csv_uploads/extra_columns.csv create mode 100644 tests/csv_uploads/malformed.csv create mode 100644 tests/csv_uploads/missing_columns.csv create mode 100644 tests/csv_uploads/missing_session_id_value.csv create mode 100644 tests/csv_uploads/not_a_csv.txt create mode 100644 tests/csv_uploads/valid_discussion.csv create mode 100644 tests/csv_uploads/valid_story.csv create mode 100644 tests/kafka_events/create/create_discussion_multi_statement_array.json create mode 100644 tests/kafka_events/create/create_discussion_multi_theme_llm.json create mode 100644 tests/kafka_events/create/create_discussion_multi_theme_local.json create mode 100644 tests/kafka_events/create/create_story_multi_barrier_single_theme.json diff --git a/tests/csv_uploads/empty.csv b/tests/csv_uploads/empty.csv new file mode 100644 index 0000000..e69de29 diff --git a/tests/csv_uploads/extra_columns.csv b/tests/csv_uploads/extra_columns.csv new file mode 100644 index 0000000..74f1aab --- /dev/null +++ b/tests/csv_uploads/extra_columns.csv @@ -0,0 +1,2 @@ +id,Title,User name,Designation,Location,District,Organization,Report Created At,Objective,Challenges,Action Steps,Impact,Duration,Blurb,masked_blurb,Content,masked_content,Images,Pdf,Transcript Link,Session ID,pri_member_info,school_representative_info,session___i +5003,Extra unexpected columns,Test User,Teacher,"Patna, Bihar",Patna,Mitra Foundation,2026-07-20,Test objective for extra-column validation.,Test challenge.,Test action step.,Test impact.,1 month,,,,,,,,sess-story-5003,unexpected,unexpected,unexpected diff --git a/tests/csv_uploads/malformed.csv b/tests/csv_uploads/malformed.csv new file mode 100644 index 0000000..92dbb91 --- /dev/null +++ b/tests/csv_uploads/malformed.csv @@ -0,0 +1,2 @@ +id,Title,"Unterminated quote field +5006,"Broken row without closing quote diff --git a/tests/csv_uploads/missing_columns.csv b/tests/csv_uploads/missing_columns.csv new file mode 100644 index 0000000..cab689f --- /dev/null +++ b/tests/csv_uploads/missing_columns.csv @@ -0,0 +1,2 @@ +id,Title,User name,Designation,Location,District,Organization,Report Created At,Objective,Challenges,Action Steps,Impact,Duration,Blurb,masked_blurb,Content,masked_content,Images,Pdf,Transcript Link +5002,Missing Session ID column,Test User,Teacher,"Patna, Bihar",Patna,Mitra Foundation,2026-07-20,Test objective for missing-column validation.,Test challenge.,Test action step.,Test impact.,1 month,,,,,,, diff --git a/tests/csv_uploads/missing_session_id_value.csv b/tests/csv_uploads/missing_session_id_value.csv new file mode 100644 index 0000000..09cec22 --- /dev/null +++ b/tests/csv_uploads/missing_session_id_value.csv @@ -0,0 +1,2 @@ +id,Title,User name,Designation,Location,District,Organization,Report Created At,Objective,Challenges,Action Steps,Impact,Duration,Blurb,masked_blurb,Content,masked_content,Images,Pdf,Transcript Link,Session ID +5004,Blank Session ID value,Test User,Teacher,"Patna, Bihar",Patna,Mitra Foundation,2026-07-20,Test objective for blank session id.,Test challenge.,Test action step.,Test impact.,1 month,,,,,,,, diff --git a/tests/csv_uploads/not_a_csv.txt b/tests/csv_uploads/not_a_csv.txt new file mode 100644 index 0000000..d3682a5 --- /dev/null +++ b/tests/csv_uploads/not_a_csv.txt @@ -0,0 +1,2 @@ +id,Title +5005,Wrong extension test diff --git a/tests/csv_uploads/valid_discussion.csv b/tests/csv_uploads/valid_discussion.csv new file mode 100644 index 0000000..7b7cb04 --- /dev/null +++ b/tests/csv_uploads/valid_discussion.csv @@ -0,0 +1,2 @@ +id,Title,User name,User Location,District,Participant Count,Men,Women,Children,Date of Discussion,Organization,Challenges,Solutions,Author,Language,Report Created At,Transcript Link,Image Urls,PDF Urls,Session ID +6001,Community discussion on school infrastructure,Ravi Singh,"Ranchi, Jharkhand",Ranchi,12,5,6,1,2026-07-18,Mitra Foundation,Lack of clean drinking water in the school premises.,Proposed installing a water filtration unit with community funding.,Ravi Singh,en,2026-07-18,https://example.com/transcripts/discussion-6001,https://example.com/images/discussion-6001-1.jpg,https://example.com/reports/discussion-6001.pdf,sess-discussion-6001 diff --git a/tests/csv_uploads/valid_story.csv b/tests/csv_uploads/valid_story.csv new file mode 100644 index 0000000..5c4fa78 --- /dev/null +++ b/tests/csv_uploads/valid_story.csv @@ -0,0 +1,2 @@ +id,Title,User name,Designation,Location,District,Organization,Report Created At,Objective,Challenges,Action Steps,Impact,Duration,Blurb,masked_blurb,Content,masked_content,Images,Pdf,Transcript Link,Session ID +5001,Improving classroom engagement,Anita Kumar,Teacher,"Patna, Bihar",Patna,Mitra Foundation,2026-07-20,Improve classroom engagement through storytelling activities.,Students were disengaged during long lecture sessions.,Introduced interactive storytelling exercises weekly.,Noticeable improvement in classroom participation.,3 months,A short story about reviving classroom engagement.,,Full narrative content describing the intervention and its outcomes.,,,https://example.com/reports/story-5001.pdf,https://example.com/transcripts/story-5001,sess-story-5001 diff --git a/tests/kafka_events/create/create_discussion_multi_statement_array.json b/tests/kafka_events/create/create_discussion_multi_statement_array.json new file mode 100644 index 0000000..40ee414 --- /dev/null +++ b/tests/kafka_events/create/create_discussion_multi_statement_array.json @@ -0,0 +1,36 @@ +{ + "submissionId": 9004, + "submissionType": "discussion", + "sessionId": "test9004multistatement01", + "tenantCode": "mitra", + "eventType": "create", + "eventPublishedAt": "2026-07-16T10:15:00.000Z", + "tags": { + "state": "Bihar", + "district": "Patna", + "organization": "Sahyogi Sanstha", + "programId": "8baf962e-7c52-46cd-9787-16e3ee4d4121", + "programName": "Shiksha Chaupals", + "leaderCategoryId": "96be72cf-aa0a-4bd3-b363-49271254177a", + "leaderCategoryName": "Women leader (WL)" + }, + "data": { + "title": "Array iteration regression test — 3 independent single-theme challenges (expected: 3 separate Standard rows, each single-themed, no delimiter splitting)", + "userId": "9004", + "userName": "Test User", + "designation": "Women leader", + "submissionDate": "2026-07-16T10:15:00Z", + "imageUrls": [], + "pdfUrls": null, + "transcriptLink": null, + "challenges": [ + "Our school children are not given books on time.", + "There is a shortage of teachers in the school, due to which subject-wise studies are not done.", + "The school is very far from the village." + ], + "solutions": [], + "participantsData": [], + "author": "9004", + "language": "en" + } +} diff --git a/tests/kafka_events/create/create_discussion_multi_theme_llm.json b/tests/kafka_events/create/create_discussion_multi_theme_llm.json new file mode 100644 index 0000000..1566cf7 --- /dev/null +++ b/tests/kafka_events/create/create_discussion_multi_theme_llm.json @@ -0,0 +1,34 @@ +{ + "submissionId": 9002, + "submissionType": "discussion", + "sessionId": "test9002multithemellm001", + "tenantCode": "mitra", + "eventType": "create", + "eventPublishedAt": "2026-07-16T10:05:00.000Z", + "tags": { + "state": "Bihar", + "district": "Patna", + "organization": "Sahyogi Sanstha", + "programId": "8baf962e-7c52-46cd-9787-16e3ee4d4121", + "programName": "Shiksha Chaupals", + "leaderCategoryId": "96be72cf-aa0a-4bd3-b363-49271254177a", + "leaderCategoryName": "Women leader (WL)" + }, + "data": { + "title": "Multi-theme test — addiction + safety, phrased abstractly (expected: below local threshold, LLM fallback finds 2 themes)", + "userId": "9002", + "userName": "Test User", + "designation": "Women leader", + "submissionDate": "2026-07-16T10:05:00Z", + "imageUrls": [], + "pdfUrls": null, + "transcriptLink": null, + "challenges": [ + "Since her father comes home intoxicated most nights and stays out late, the girl feels too frightened to walk back home from school alone once evening falls." + ], + "solutions": [], + "participantsData": [], + "author": "9002", + "language": "en" + } +} diff --git a/tests/kafka_events/create/create_discussion_multi_theme_local.json b/tests/kafka_events/create/create_discussion_multi_theme_local.json new file mode 100644 index 0000000..ca0d001 --- /dev/null +++ b/tests/kafka_events/create/create_discussion_multi_theme_local.json @@ -0,0 +1,34 @@ +{ + "submissionId": 9001, + "submissionType": "discussion", + "sessionId": "test9001multithemelocal01", + "tenantCode": "mitra", + "eventType": "create", + "eventPublishedAt": "2026-07-16T10:00:00.000Z", + "tags": { + "state": "Bihar", + "district": "Patna", + "organization": "Sahyogi Sanstha", + "programId": "8baf962e-7c52-46cd-9787-16e3ee4d4121", + "programName": "Shiksha Chaupals", + "leaderCategoryId": "96be72cf-aa0a-4bd3-b363-49271254177a", + "leaderCategoryName": "Women leader (WL)" + }, + "data": { + "title": "Multi-theme test — distance + parental negligence (expected: local embedding match, 2 themes)", + "userId": "9001", + "userName": "Test User", + "designation": "Women leader", + "submissionDate": "2026-07-16T10:00:00Z", + "imageUrls": [], + "pdfUrls": null, + "transcriptLink": null, + "challenges": [ + "There is no school in the village after middle school, so children have to go far away, and due to the negligence of parents, both parents and children do not want to study further." + ], + "solutions": [], + "participantsData": [], + "author": "9001", + "language": "en" + } +} diff --git a/tests/kafka_events/create/create_story_multi_barrier_single_theme.json b/tests/kafka_events/create/create_story_multi_barrier_single_theme.json new file mode 100644 index 0000000..328f2fc --- /dev/null +++ b/tests/kafka_events/create/create_story_multi_barrier_single_theme.json @@ -0,0 +1,34 @@ +{ + "submissionId": 9003, + "submissionType": "story", + "sessionId": "test9003storysinglethem01", + "tenantCode": "mitra", + "eventType": "create", + "eventPublishedAt": "2026-07-16T10:10:00.000Z", + "tags": { + "state": "Bihar", + "district": "Patna", + "organization": "Sahyogi Sanstha", + "programId": "8baf962e-7c52-46cd-9787-16e3ee4d4121", + "programName": "Shiksha Chaupals", + "leaderCategoryId": "96be72cf-aa0a-4bd3-b363-49271254177a", + "leaderCategoryName": "Women leader (WL)" + }, + "data": { + "title": "Story test — same multi-barrier text as create_discussion_multi_theme_local.json (expected: single theme, story never multi-maps)", + "userId": "9003", + "userName": "Test User", + "designation": "Facilitator", + "submissionDate": "2026-07-16T10:10:00Z", + "imageUrls": [], + "pdfUrls": null, + "transcriptLink": null, + "objective": "There is no school in the village after middle school, so children have to go far away, and due to the negligence of parents, both parents and children do not want to study further.", + "challenges": [], + "actionSteps": [], + "impact": "", + "duration": "", + "blurb": "", + "content": "" + } +} From 69e559e45f7b96a0dab6f6d2936d6e7a62d1db45 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:32:55 +0530 Subject: [PATCH 03/10] updated docker-compose.yaml and .env.example --- .env.example | 7 ++ docker-compose.yaml | 214 ++++++++++++++++++++++---------------------- 2 files changed, 114 insertions(+), 107 deletions(-) diff --git a/.env.example b/.env.example index 2f4c34c..c7f9e04 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,12 @@ # Example .env file for analytics_service +# Docker Compose deployment mode toggle — read automatically by the `docker +# compose` CLI (not the app itself). "split" = 3 separate containers +# (analytics-web/consumer/worker, fault-isolated, default). "single" = one +# container running all three as separate processes (see run_all.sh). +# Only one should be active at a time — both bind host port 8000. +COMPOSE_PROFILES=split + # Logging Configuration LOG_DIR=logs LOG_LEVEL=INFO diff --git a/docker-compose.yaml b/docker-compose.yaml index 94a10a5..71b9282 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -38,120 +38,120 @@ services: depends_on: - temporal - # kafka: - # image: apache/kafka:3.9.0 - # container_name: kafka - # ports: - # - "29092:29092" # host-facing listener only — do NOT also publish 9092 - # environment: - # KAFKA_NODE_ID: 1 - # KAFKA_PROCESS_ROLES: broker,controller - # KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,PLAINTEXT_HOST://:29092 - # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 - # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - # KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - # KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - # KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 - # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - # KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - # CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk" - # volumes: - # - kafka_data:/var/lib/kafka/data - # healthcheck: - # test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server kafka:9092 || exit 1"] - # interval: 10s - # timeout: 5s - # retries: 10 - # start_period: 20s + kafka: + image: apache/kafka:3.9.0 + container_name: kafka + ports: + - "29092:29092" # host-facing listener only — do NOT also publish 9092 + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,PLAINTEXT_HOST://:29092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk" + volumes: + - kafka_data:/var/lib/kafka/data + healthcheck: + test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server kafka:9092 || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s - # # --- Split mode: one container per service (default — fault isolation, - # # independent scaling/restarts). Active when COMPOSE_PROFILES=split (see .env). - # analytics-web: - # profiles: ["split"] - # build: . - # image: elevate-analytics:latest - # env_file: .env - # environment: - # DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db - # TEMPORAL_HOST: temporal:7233 - # KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - # extra_hosts: - # - "host.docker.internal:host-gateway" - # depends_on: - # temporal: - # condition: service_healthy - # kafka: - # condition: service_healthy - # ports: - # - "8000:8000" - # volumes: - # - ./logs:/app/logs - # command: ["--mode", "web"] + # --- Split mode: one container per service (default — fault isolation, + # independent scaling/restarts). Active when COMPOSE_PROFILES=split (see .env). + analytics-web: + profiles: ["split"] + build: . + image: elevate-analytics:latest + env_file: .env + environment: + DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + TEMPORAL_HOST: temporal:7233 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + temporal: + condition: service_healthy + kafka: + condition: service_healthy + ports: + - "8000:8000" + volumes: + - ./logs:/app/logs + command: ["--mode", "web"] - # analytics-consumer: - # profiles: ["split"] - # build: . - # image: elevate-analytics:latest - # env_file: .env - # environment: - # DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db - # TEMPORAL_HOST: temporal:7233 - # KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - # extra_hosts: - # - "host.docker.internal:host-gateway" - # depends_on: - # temporal: - # condition: service_healthy - # kafka: - # condition: service_healthy - # volumes: - # - ./logs:/app/logs - # command: ["--mode", "consumer"] + analytics-consumer: + profiles: ["split"] + build: . + image: elevate-analytics:latest + env_file: .env + environment: + DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + TEMPORAL_HOST: temporal:7233 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + temporal: + condition: service_healthy + kafka: + condition: service_healthy + volumes: + - ./logs:/app/logs + command: ["--mode", "consumer"] - # analytics-worker: - # profiles: ["split"] - # build: . - # image: elevate-analytics:latest - # env_file: .env - # environment: - # DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db - # TEMPORAL_HOST: temporal:7233 - # KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - # extra_hosts: - # - "host.docker.internal:host-gateway" - # depends_on: - # temporal: - # condition: service_healthy - # kafka: - # condition: service_healthy - # volumes: - # - ./logs:/app/logs - # command: ["--mode", "worker"] + analytics-worker: + profiles: ["split"] + build: . + image: elevate-analytics:latest + env_file: .env + environment: + DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + TEMPORAL_HOST: temporal:7233 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + temporal: + condition: service_healthy + kafka: + condition: service_healthy + volumes: + - ./logs:/app/logs + command: ["--mode", "worker"] # --- Single mode: one container, all three services as separate processes # (run_all.sh). Active when COMPOSE_PROFILES=single (see .env). Do not run # this alongside the split services above — both bind host port 8000. - # analytics-all: - # profiles: ["single"] - # build: . - # image: elevate-analytics:latest - # env_file: .env - # environment: - # DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db - # TEMPORAL_HOST: temporal:7233 - # KAFKA_BOOTSTRAP_SERVERS: kafka:9092 - # extra_hosts: - # - "host.docker.internal:host-gateway" - # depends_on: - # temporal: - # condition: service_healthy - # kafka: - # condition: service_healthy - # ports: - # - "8000:8000" - # volumes: - # - ./logs:/app/logs - # entrypoint: ["/app/run_all.sh"] + analytics-all: + profiles: ["single"] + build: . + image: elevate-analytics:latest + env_file: .env + environment: + DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + TEMPORAL_HOST: temporal:7233 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + temporal: + condition: service_healthy + kafka: + condition: service_healthy + ports: + - "8000:8000" + volumes: + - ./logs:/app/logs + entrypoint: ["/app/run_all.sh"] volumes: kafka_data: From 7886d656c5dec5946c88a37be95304049ff80a0a Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Thu, 23 Jul 2026 12:31:52 +0530 Subject: [PATCH 04/10] FIX: Serialize schema initialization across processes with a Postgres advisory lock run_all.sh's single-container mode starts web/consumer/worker as three separate OS processes, each independently calling db.connect()/initialize_schema() at startup. The existing asyncio.Lock only serializes coroutines within one process, so on a cold database the three processes raced on the same schema DDL and one crashed with a Postgres catalog collision (duplicate key on pg_type), killing the whole container. Discovered while load-testing the single-container profile. Co-Authored-By: Claude Sonnet 5 --- app/database/db.py | 63 +++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/app/database/db.py b/app/database/db.py index 0212305..19d8308 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -27,32 +27,43 @@ async def initialize_schema(self) -> None: raise RuntimeError("Database pool is not initialized. Call connect() first.") async with self.pool.acquire() as conn: - if settings.RESET_DB: - if settings.ENVIRONMENT.lower().strip() == "development": - await conn.execute("DROP SCHEMA IF EXISTS public CASCADE;") - await conn.execute("CREATE SCHEMA public;") - logger.warning("Database schema reset requested; dropped and recreated public schema.") - else: - logger.error( - f"RESET_DB=True was requested but ENVIRONMENT={settings.ENVIRONMENT!r} is not " - "'development' — refusing to drop the schema. Set ENVIRONMENT=development if " - "this is intentional." - ) - - schema_sql = SCHEMA_FILE.read_text(encoding="utf-8") - schema_sql = schema_sql.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ") - schema_sql = schema_sql.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ") - await conn.execute(schema_sql) - - # Always run the seed script to keep prompts in sync with seed_prompts.sql - seed_sql = SEED_PROMPTS_FILE.read_text(encoding="utf-8") - await conn.execute(seed_sql) - - # Always run the themes seed script to seed initial approved taxonomies - seed_themes_sql = SEED_THEMES_FILE.read_text(encoding="utf-8") - await conn.execute(seed_themes_sql) - - logger.info("Database schema initialized successfully.") + # web/consumer/worker each run as separate processes (run_all.sh) and + # every one of them calls connect()/initialize_schema() independently + # at startup — the asyncio.Lock below only serializes coroutines within + # one process, so without a cross-process lock they race on the same + # CREATE TABLE/TYPE DDL and one hits a Postgres catalog collision (e.g. + # duplicate key on pg_type) even with "IF NOT EXISTS", since the + # existence check and creation aren't atomic across concurrent sessions. + await conn.execute("SELECT pg_advisory_lock(727384910)") + try: + if settings.RESET_DB: + if settings.ENVIRONMENT.lower().strip() == "development": + await conn.execute("DROP SCHEMA IF EXISTS public CASCADE;") + await conn.execute("CREATE SCHEMA public;") + logger.warning("Database schema reset requested; dropped and recreated public schema.") + else: + logger.error( + f"RESET_DB=True was requested but ENVIRONMENT={settings.ENVIRONMENT!r} is not " + "'development' — refusing to drop the schema. Set ENVIRONMENT=development if " + "this is intentional." + ) + + schema_sql = SCHEMA_FILE.read_text(encoding="utf-8") + schema_sql = schema_sql.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ") + schema_sql = schema_sql.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ") + await conn.execute(schema_sql) + + # Always run the seed script to keep prompts in sync with seed_prompts.sql + seed_sql = SEED_PROMPTS_FILE.read_text(encoding="utf-8") + await conn.execute(seed_sql) + + # Always run the themes seed script to seed initial approved taxonomies + seed_themes_sql = SEED_THEMES_FILE.read_text(encoding="utf-8") + await conn.execute(seed_themes_sql) + + logger.info("Database schema initialized successfully.") + finally: + await conn.execute("SELECT pg_advisory_unlock(727384910)") async def connect(self) -> None: """ From 8e37f843a421d5d6b7a56d1f31dd9ce04bd863e8 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Thu, 23 Jul 2026 13:42:12 +0530 Subject: [PATCH 05/10] FIX: Make the asyncpg connection pool size configurable, raise the effective default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Load-testing 200 concurrent real-time submissions against the hardcoded min_size=2/max_size=10 pool caused nearly the entire batch (196/200) to stall indefinitely in "processing" — activities queued for a DB connection and were cancelled rather than failing fast, since the pool was drastically undersized for that concurrency. Exposed as DATABASE_POOL_MIN_SIZE/DATABASE_POOL_MAX_SIZE settings so it's tunable per environment without a code change; re-running the same 200-event batch with the pool raised to 5/50 completed 198/200 in 6 minutes (the other 2 failed on an unrelated malformed-LLM-JSON response, not a pool issue). Co-Authored-By: Claude Sonnet 5 --- .env.example | 6 ++++++ app/config.py | 7 +++++++ app/database/db.py | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index c7f9e04..c6bc259 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,12 @@ ENVIRONMENT=development # Database Configuration DATABASE_URL=postgresql://postgres:postgres@localhost:5432/temporal RESET_DB=false +# asyncpg pool size — this is the real concurrency ceiling for DB-touching +# activities. Raise DATABASE_POOL_MAX_SIZE for higher-throughput environments; +# too small and high-concurrency batches stall on pool.acquire() rather than +# failing fast (load-tested: 10 stalls hard around ~150-200 concurrent events). +DATABASE_POOL_MIN_SIZE=2 +DATABASE_POOL_MAX_SIZE=10 # Orchestration Mode: 'real-time' or 'batch' PROCESSING_MODE=real-time diff --git a/app/config.py b/app/config.py index 9d6db17..ba44234 100644 --- a/app/config.py +++ b/app/config.py @@ -12,6 +12,13 @@ class Settings(BaseSettings): # Database Configuration DATABASE_URL: str = Field(default="postgresql://postgres:postgres@localhost:5432/temporal") + # asyncpg connection pool bounds — this is the real concurrency ceiling for + # DB-touching activities (insert/update submission, llm_logs, etc). Sized too + # small and high-concurrency batches (e.g. 200+ simultaneous real-time + # submissions) stall almost entirely on pool.acquire() rather than failing + # fast, since activities queue for a connection instead of erroring out. + DATABASE_POOL_MIN_SIZE: int = Field(default=2, gt=0) + DATABASE_POOL_MAX_SIZE: int = Field(default=10, gt=0) # Orchestration Mode: 'real-time' or 'batch' PROCESSING_MODE: str = Field(default="real-time") diff --git a/app/database/db.py b/app/database/db.py index 19d8308..f13b75a 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -82,8 +82,8 @@ async def connect(self) -> None: try: self.pool = await asyncpg.create_pool( dsn=settings.DATABASE_URL, - min_size=2, - max_size=10 + min_size=settings.DATABASE_POOL_MIN_SIZE, + max_size=settings.DATABASE_POOL_MAX_SIZE ) await self.initialize_schema() logger.info("Database connection pool established successfully.") From 4d752d35d3f13b45f7682c633dcf2e26399a4cc5 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:48:57 +0530 Subject: [PATCH 06/10] perf(worker): optimize Temporal worker thread pool and PyTorch threading - Set on Temporal worker. - Replace default asyncio thread pool executor with custom ThreadPoolExecutor sized to worker limits. - Cap PyTorch CPU intra-op threads to 1 to avoid thread thrashing during concurrent embeddings. --- app/temporal/worker.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/app/temporal/worker.py b/app/temporal/worker.py index a398317..d796563 100644 --- a/app/temporal/worker.py +++ b/app/temporal/worker.py @@ -1,5 +1,7 @@ import asyncio +import concurrent.futures import logging +import torch from temporalio.client import Client from temporalio.worker import Worker @@ -32,6 +34,32 @@ async def start_worker(): """ Connects to Temporal server and listens on the configured task queue. """ + # Python's default asyncio thread pool executor is capped at + # min(32, cpu_count + 4) — far smaller than our configured worker + # concurrency, and every activity's blocking work (LLM calls via urllib, + # image download/blur/upload, PDF extraction, embeddings) runs through + # asyncio.to_thread(), which uses this default executor unless overridden. + # Without sizing it explicitly, WORKER_MAX_CONCURRENT_ACTIVITIES is a + # ceiling Temporal never actually reaches — most "concurrent" activities + # just queue for a free thread instead of doing real work (load-tested: + # this silently capped real throughput to ~12-way parallelism on an + # 8-core box, not the 40 configured at the Temporal level). + asyncio.get_running_loop().set_default_executor( + concurrent.futures.ThreadPoolExecutor(max_workers=settings.WORKER_MAX_CONCURRENT_ACTIVITIES) + ) + + # PyTorch defaults to using EVERY available CPU core for its own internal + # BLAS/linear-algebra threading on each individual call (SentenceTransformer + # embeddings here) — fine for a single request at a time, catastrophic once + # many activities call into it concurrently. Load-tested: with this unset, + # concurrent embedding calls under real load turned a sub-second operation + # into 15+ minutes (dozens of activities each trying to claim all 8 cores + # for their own inference call, thrashing on context switches instead of + # doing work). Capping PyTorch's OWN thread count to 1 makes + # WORKER_MAX_CONCURRENT_ACTIVITIES the only source of parallelism, instead + # of the two multiplying against each other. + torch.set_num_threads(1) + # Initialize database connection pool await db.connect() @@ -67,7 +95,8 @@ async def start_worker(): client, task_queue=settings.TEMPORAL_QUEUE, workflows=workflows, - activities=activities + activities=activities, + max_concurrent_activities=settings.WORKER_MAX_CONCURRENT_ACTIVITIES, ) # Register daily batch schedules if configured for batch mode From f52203d1830c9ffa8156baa60f5c1b78f2e4ccf7 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:50:53 +0530 Subject: [PATCH 07/10] perf(temporal): parallelize image processing and fix status handling - Parallelize image face-blurring with asyncio.gather and semaphore concurrency bounds. - Defer DB connection acquisition in deface_blur_activity to avoid holding idle connections during image processing. - Remove premature submission status updates from pii_and_abusive_activity. --- app/services/classifier.py | 29 ++++++++++++++++++++----- app/temporal/csv_processing_activity.py | 22 ++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/app/services/classifier.py b/app/services/classifier.py index 001891d..758b186 100644 --- a/app/services/classifier.py +++ b/app/services/classifier.py @@ -5,6 +5,7 @@ and approved themes, then computes cosine similarity to find the best match. """ import logging +import threading from typing import Dict, Any, List, Optional, Tuple import numpy as np @@ -17,16 +18,32 @@ # Module-level model cache — loaded once per worker process _model: Optional[SentenceTransformer] = None +# Real OS-thread lock, not asyncio.Lock — this is called from separate threads +# via asyncio.to_thread (build_theme_embeddings/get_theme_similarities), not +# concurrently within one event loop. +_model_lock = threading.Lock() def _get_model() -> SentenceTransformer: - """Lazily load the sentence transformer model (cached at module level).""" + """ + Lazily load the sentence transformer model (cached at module level). + Thread-safe: double-checked locking, since multiple concurrent activities + call this from separate OS threads. Without the lock, several threads can + all see _model is None at once and each construct their own + SentenceTransformer concurrently — that's not safe (load-tested: it + corrupts PyTorch's internal state with "NotImplementedError: Cannot copy + out of meta tensor; no data" under concurrent load). The un-locked fast + path below keeps the common case (already loaded) lock-free. + """ global _model - if _model is None: - model_name = settings.EMBEDDING_MODEL_NAME - logger.info(f"Loading SentenceTransformer model '{model_name}'...") - _model = SentenceTransformer(model_name) - logger.info(f"SentenceTransformer model '{model_name}' loaded successfully.") + if _model is not None: + return _model + with _model_lock: + if _model is None: + model_name = settings.EMBEDDING_MODEL_NAME + logger.info(f"Loading SentenceTransformer model '{model_name}'...") + _model = SentenceTransformer(model_name) + logger.info(f"SentenceTransformer model '{model_name}' loaded successfully.") return _model diff --git a/app/temporal/csv_processing_activity.py b/app/temporal/csv_processing_activity.py index e1956cd..5a7b997 100644 --- a/app/temporal/csv_processing_activity.py +++ b/app/temporal/csv_processing_activity.py @@ -1,6 +1,7 @@ import asyncio import json import logging +import threading import uuid from datetime import datetime from typing import Any, Dict, List, Optional @@ -18,16 +19,25 @@ logger = logging.getLogger("analytics_service.temporal.csv_processing_activity") _producer: Optional[Producer] = None +# Real OS-thread lock, not asyncio.Lock — _get_producer() is called from +# separate threads via asyncio.to_thread (_push_rows_sync), not concurrently +# within one event loop. Without this, concurrent CsvProcessingWorkflow child +# workflows could each construct their own Producer at once (see the identical +# fix + reasoning in app/services/classifier.py's _get_model()). +_producer_lock = threading.Lock() def _get_producer() -> Producer: global _producer - if _producer is None: - _producer = Producer({ - "bootstrap.servers": settings.KAFKA_BOOTSTRAP_SERVERS, - "acks": "all", - "enable.idempotence": True, - }) + if _producer is not None: + return _producer + with _producer_lock: + if _producer is None: + _producer = Producer({ + "bootstrap.servers": settings.KAFKA_BOOTSTRAP_SERVERS, + "acks": "all", + "enable.idempotence": True, + }) return _producer From 991a61d7d376be2b64deab1384e139d025b02a44 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:51:52 +0530 Subject: [PATCH 08/10] perf(temporal): parallelize image processing and fix status handling - Parallelize image face-blurring with and semaphore concurrency bounds. - Defer DB connection acquisition in to avoid holding idle connections during image processing. - Remove premature submission status updates from . --- app/temporal/deface_blur_activity.py | 241 +++++++++++++++-------- app/temporal/pii_and_abusive_activity.py | 15 +- 2 files changed, 171 insertions(+), 85 deletions(-) diff --git a/app/temporal/deface_blur_activity.py b/app/temporal/deface_blur_activity.py index 04f3095..1e8d479 100644 --- a/app/temporal/deface_blur_activity.py +++ b/app/temporal/deface_blur_activity.py @@ -3,8 +3,10 @@ import urllib.request import urllib.parse import asyncio +import concurrent.futures +import functools from pathlib import Path -from typing import Dict, Any, List +from typing import Dict, Any from temporalio import activity from app.config import settings @@ -19,107 +21,184 @@ DOWNLOADS_DIR = BASE_DIR / "downloads" OUTPUTS_DIR = BASE_DIR / "outputs" +# Dedicated executor for image download/blur/upload — kept separate from the +# worker's general activity thread pool (sized to WORKER_MAX_CONCURRENT_ACTIVITIES +# in app/temporal/worker.py) so image-processing concurrency can be tuned +# independently, without this activity's fan-out stealing thread-pool capacity +# from LLM/embedding activities or vice versa. Size via IMAGE_EXECUTOR_MAX_WORKERS +# (settings) — downloads/uploads are I/O-bound and benefit from more concurrency +# than there are cores, but the face-blur step itself is CPU-bound and gains +# nothing past that; the default splits the difference rather than optimizing +# purely for either side. +_IMAGE_EXECUTOR = concurrent.futures.ThreadPoolExecutor( + max_workers=settings.IMAGE_EXECUTOR_MAX_WORKERS, + thread_name_prefix="image-blur", +) + +_blur_semaphore: asyncio.Semaphore = None +_blur_semaphore_loop = None + + +def _get_blur_semaphore() -> asyncio.Semaphore: + # Lazily (re)created against whichever event loop is currently running — + # a module-level asyncio.Semaphore() constructed at import time would bind + # to the wrong loop (mirrors the same pattern app/database/db.py already + # uses for its own connect lock, for the same reason). Size via + # BLUR_CONCURRENCY_LIMIT (settings) — caps how many face-blur subprocesses + # run at once ACROSS THE WHOLE WORKER PROCESS, not per submission. + # anonymize_face() shells out to the deface CLI, which spawns a fresh + # process and loads its own ONNX model weights on every call: memory-heavy, + # unlike the download/upload legs. Load-tested: letting this scale with + # submission concurrency (instead of a small global cap) got a deface + # subprocess OOM-killed (exit code -9) under concurrent load. This should + # be sized against available memory headroom, NOT submission count, worker + # concurrency, or CPU count. + global _blur_semaphore, _blur_semaphore_loop + current_loop = asyncio.get_running_loop() + if _blur_semaphore is None or _blur_semaphore_loop is not current_loop: + _blur_semaphore = asyncio.Semaphore(settings.BLUR_CONCURRENCY_LIMIT) + _blur_semaphore_loop = current_loop + return _blur_semaphore + + +async def _run_in_image_executor(func, *args, **kwargs): + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_IMAGE_EXECUTOR, functools.partial(func, *args, **kwargs)) + def _download_file(url: str, filename: str) -> Path: DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True) local_path = DOWNLOADS_DIR / filename logger.info(f"Downloading {url} to {local_path}") - + with urllib.request.urlopen(url, timeout=60) as response: with open(local_path, "wb") as f: f.write(response.read()) return local_path +async def _process_one_image(submission_id: str, tenant_code: str, sub_type: str, i: int, url: Any) -> Dict[str, Any]: + """ + Downloads, blurs, and uploads a single image. Runs concurrently with the + submission's other images (see deface_blur_activity), bounded by + PER_SUBMISSION_IMAGE_CONCURRENCY setting, instead of the previous one-at-a-time loop. + """ + url_str = str(url).strip() + if not (url_str.startswith("http://") or url_str.startswith("https://")): + base_url = settings.MEDIA_BASE_URL + if not base_url: + raise ValueError("Relative image URL encountered but MEDIA_BASE_URL is not configured.") + resolved_url = urllib.parse.urljoin(base_url.rstrip("/") + "/", url_str.lstrip("/")) + logger.info(f"Reconstructed absolute URL for download: {resolved_url} (from relative path: {url_str})") + else: + resolved_url = url_str + + parsed_path = urllib.parse.urlparse(resolved_url).path + parts = [p for p in parsed_path.split("/") if p] + if len(parts) >= 2: + actual_name = f"{parts[-2]}/{parts[-1]}" + else: + actual_name = parts[-1] if parts else f"{submission_id}_{i}.jpg" + + ext = os.path.splitext(parsed_path)[1] + if not ext: + ext = ".jpg" + filename = f"{submission_id}_{tenant_code}_{i}{ext}" + + local_path = DOWNLOADS_DIR / filename + output_path = OUTPUTS_DIR / f"blurred_{filename}" + + try: + # 1. Download file locally + await _run_in_image_executor(_download_file, resolved_url, filename) + + # 2. Deface/Blur image — gated globally (see BLUR_CONCURRENCY_LIMIT setting), + # unlike the download/upload legs, since this is the memory-heavy step. + OUTPUTS_DIR.mkdir(parents=True, exist_ok=True) + async with _get_blur_semaphore(): + await _run_in_image_executor( + anonymize_face, + input_path=str(local_path), + output_path=str(output_path), + ) + + # 3. Upload to GCP Storage + if "story" in sub_type: + blob_prefix = settings.STORY_BLOB or "story_blurred_image" + else: + blob_prefix = settings.DISCUSSION_BLOB or "dicussion_blurred_image" + + blob_name = f"{blob_prefix}/{actual_name}" + public_url = await _run_in_image_executor(upload_to_gcp, str(output_path), blob_name) + + return {"relative_url": parsed_path, "public_url": public_url} + + except Exception as e: + logger.error(f"Failed face blurring for {resolved_url}: {e}") + raise + finally: + # Clean up local temporary files under all conditions (prevent disk leakage) + if local_path.exists(): + try: + local_path.unlink() + except Exception as clean_err: + logger.warning(f"Failed to delete temp file {local_path}: {clean_err}") + if output_path.exists(): + try: + output_path.unlink() + except Exception as clean_err: + logger.warning(f"Failed to delete temp file {output_path}: {clean_err}") + + @activity.defn async def deface_blur_activity(params: Dict[str, Any]) -> Dict[str, Any]: """ Temporal activity that downloads and runs local OpenCV/ONNX face blurring on ingestion images, - then uploads the result to GCP Storage. + then uploads the result to GCP Storage. A submission's images process concurrently with each + other (bounded by PER_SUBMISSION_IMAGE_CONCURRENCY setting) instead of one at a time — load-tested, + the sequential version made image_blur ~66% of a submission's total processing time. """ submission_id = params["submission_id"] tenant_code = params["tenant_code"] + # Only hold a DB connection for the brief read/write on either side of the + # actual work — never across the download/blur/upload work below. That work + # is network- and CPU-bound (can run for minutes); holding a pool + # connection idle for that whole time is what let a handful of concurrent + # submissions pin most of the pool, starving every other activity that + # needs a quick connection. async with db.pool.acquire() as conn: sub_type, payload = await get_submission_type_and_payload(conn, submission_id, tenant_code) - - image_urls = payload.get("image_urls") - if not image_urls: - return {"status": "skipped", "reason": "no image urls available"} - - blurred_local_paths = [] - relative_original_urls = [] - - for i, url in enumerate(image_urls): - # Parse URL to reconstruct if it is relative (which is common in batch-mode/retries) - url_str = str(url).strip() - if not (url_str.startswith("http://") or url_str.startswith("https://")): - base_url = settings.MEDIA_BASE_URL - if not base_url: - raise ValueError("Relative image URL encountered but MEDIA_BASE_URL is not configured.") - resolved_url = urllib.parse.urljoin(base_url.rstrip("/") + "/", url_str.lstrip("/")) - logger.info(f"Reconstructed absolute URL for download: {resolved_url} (from relative path: {url_str})") - else: - resolved_url = url_str - - parsed_path = urllib.parse.urlparse(resolved_url).path - parts = [p for p in parsed_path.split("/") if p] - if len(parts) >= 2: - actual_name = f"{parts[-2]}/{parts[-1]}" - else: - actual_name = parts[-1] if parts else f"{submission_id}_{i}.jpg" - - relative_original_urls.append(parsed_path) - ext = os.path.splitext(parsed_path)[1] - if not ext: - ext = ".jpg" - filename = f"{submission_id}_{tenant_code}_{i}{ext}" - - local_path = DOWNLOADS_DIR / filename - output_path = OUTPUTS_DIR / f"blurred_{filename}" - - try: - # 1. Download file locally (non-blocking thread pool execution) - await asyncio.to_thread(_download_file, resolved_url, filename) - - # 2. Deface/Blur image - OUTPUTS_DIR.mkdir(parents=True, exist_ok=True) - await asyncio.to_thread( - anonymize_face, - input_path=str(local_path), - output_path=str(output_path) - ) - - # 3. Upload to GCP Storage (non-blocking thread pool execution) - if "story" in sub_type: - blob_prefix = settings.STORY_BLOB or "story_blurred_image" - else: - blob_prefix = settings.DISCUSSION_BLOB or "dicussion_blurred_image" - - blob_name = f"{blob_prefix}/{actual_name}" - - public_url = await asyncio.to_thread(upload_to_gcp, str(output_path), blob_name) - blurred_local_paths.append(public_url) - - except Exception as e: - logger.error(f"Failed face blurring for {resolved_url}: {e}") - raise - finally: - # Clean up local temporary files under all conditions (prevent disk leakage) - if local_path.exists(): - try: - local_path.unlink() - except Exception as clean_err: - logger.warning(f"Failed to delete temp file {local_path}: {clean_err}") - if output_path.exists(): - try: - output_path.unlink() - except Exception as clean_err: - logger.warning(f"Failed to delete temp file {output_path}: {clean_err}") - - # Save output paths back to DB - if blurred_local_paths or relative_original_urls: + image_urls = payload.get("image_urls") + if not image_urls: + return {"status": "skipped", "reason": "no image urls available"} + + semaphore = asyncio.Semaphore(settings.PER_SUBMISSION_IMAGE_CONCURRENCY) + + async def _bounded(i: int, url: Any) -> Dict[str, Any]: + async with semaphore: + return await _process_one_image(submission_id, tenant_code, sub_type, i, url) + + # gather() preserves input order in its results regardless of completion + # order, so blurred_local_paths/relative_original_urls below stay aligned + # with the original image_urls order exactly as the old sequential loop did. + results = await asyncio.gather( + *[_bounded(i, url) for i, url in enumerate(image_urls)], + return_exceptions=True, + ) + for r in results: + if isinstance(r, Exception): + raise r + + blurred_local_paths = [r["public_url"] for r in results] + relative_original_urls = [r["relative_url"] for r in results] + + # Save output paths back to DB — acquire fresh here rather than reusing a + # connection held since the top, since the work above may have taken + # minutes and the earlier connection would have sat idle that whole time. + if blurred_local_paths or relative_original_urls: + async with db.pool.acquire() as conn: if sub_type == "story": await conn.execute( "UPDATE story_submissions SET blur_image_urls = $3, image_urls = $4, updated_at = now() WHERE submission_id = $1 AND tenant_code = $2", @@ -131,4 +210,4 @@ async def deface_blur_activity(params: Dict[str, Any]) -> Dict[str, Any]: submission_id, tenant_code, blurred_local_paths, relative_original_urls ) - return {"status": "success", "blur_paths": blurred_local_paths} + return {"status": "success", "blur_paths": blurred_local_paths} diff --git a/app/temporal/pii_and_abusive_activity.py b/app/temporal/pii_and_abusive_activity.py index 5c306a6..85e394f 100644 --- a/app/temporal/pii_and_abusive_activity.py +++ b/app/temporal/pii_and_abusive_activity.py @@ -271,8 +271,13 @@ async def pii_and_abusive_language_detection_activity(params: Dict[str, Any]) -> meta_data=usage_meta or None, ) - # Step 6. Update the status in submissions to success - await update_submission_status(conn, submission_id, tenant_code, "success") + # Deliberately not marking the submission's overall status "success" + # here — this is only the first of several pipeline steps (thematic + # classification, image blur, and for stories, story rating still + # follow). Only the workflow's own final update_status_activity call, + # once every step has actually completed, is allowed to set the + # submission's terminal status — otherwise the row reports "success" + # while most of the pipeline hasn't run yet. return { "status": "success", @@ -310,8 +315,10 @@ async def pii_and_abusive_language_detection_activity(params: Dict[str, Any]) -> meta_data=usage_meta ) - # Update the status in submissions to failed - await update_submission_status(conn, submission_id, tenant_code, "failed") + # Not marking the submission "failed" here either — the workflow's + # own exception handler (workflows.py) already does this with the + # full per-step process_status once this exception propagates up, + # via the same raise below. except Exception as log_err: logger.error(f"Failed to log error to llm_logs: {log_err}") From 0bec42d47626fefca9ea9b66fa50ff61d82a40ae Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:53:10 +0530 Subject: [PATCH 09/10] chore(config): add worker concurrency limits and pool settings - Define WORKER_MAX_CONCURRENT_ACTIVITIES, image executor settings, and update DB pool defaults in app/config.py and .env.example. --- .env.example | 28 ++++++++++++++++++++++------ app/config.py | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index c6bc259..8114262 100644 --- a/.env.example +++ b/.env.example @@ -25,12 +25,15 @@ ENVIRONMENT=development # Database Configuration DATABASE_URL=postgresql://postgres:postgres@localhost:5432/temporal RESET_DB=false -# asyncpg pool size — this is the real concurrency ceiling for DB-touching -# activities. Raise DATABASE_POOL_MAX_SIZE for higher-throughput environments; -# too small and high-concurrency batches stall on pool.acquire() rather than -# failing fast (load-tested: 10 stalls hard around ~150-200 concurrent events). -DATABASE_POOL_MIN_SIZE=2 -DATABASE_POOL_MAX_SIZE=10 +# asyncpg pool size. Must stay comfortably UNDER your Postgres instance's own +# max_connections (with headroom for other clients/replicas) — it is NOT a +# "raise it for more throughput" knob. Load-tested: 10 stalls hard around +# ~150-200 concurrent events, but 100 against a 100-connection Postgres +# instance caused mass "sorry, too many clients already" failures — worse +# than the stall. See WORKER_MAX_CONCURRENT_ACTIVITIES below for the setting +# that actually protects this pool from an unpredictable event burst. +DATABASE_POOL_MIN_SIZE=10 +DATABASE_POOL_MAX_SIZE=50 # Orchestration Mode: 'real-time' or 'batch' PROCESSING_MODE=real-time @@ -40,6 +43,19 @@ BATCH_SIZE=100 # Temporal Configuration TEMPORAL_HOST=localhost:7233 TEMPORAL_QUEUE=analytics-processing-queue +# Caps concurrent activity execution regardless of event burst size — the +# actual protection for the DB pool. Keep at or below DATABASE_POOL_MAX_SIZE. +WORKER_MAX_CONCURRENT_ACTIVITIES=40 + +# Image processing (deface_blur_activity) concurrency — tune against actual +# server specs (CPU cores, available memory). Download/upload are cheap I/O; +# face-blur spawns a real subprocess with a fresh ONNX model load per call and +# is the memory-expensive step — keep BLUR_CONCURRENCY_LIMIT small and sized +# against available memory, not CPU count or WORKER_MAX_CONCURRENT_ACTIVITIES. +# IMAGE_EXECUTOR_MAX_WORKERS defaults to max(4, cpu_count*2) if unset. +IMAGE_EXECUTOR_MAX_WORKERS=16 +PER_SUBMISSION_IMAGE_CONCURRENCY=3 +BLUR_CONCURRENCY_LIMIT=2 # API Authentication — single shared Bearer token checked via secrets.compare_digest # in app/api/deps.py. Required (no default) — the app will not start without it. diff --git a/app/config.py b/app/config.py index ba44234..0150b0b 100644 --- a/app/config.py +++ b/app/config.py @@ -1,4 +1,5 @@ import json +import os from typing import Dict, Any, List from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -40,6 +41,24 @@ class Settings(BaseSettings): # Temporal Configuration TEMPORAL_HOST: str = Field(default="localhost:7233") TEMPORAL_QUEUE: str = Field(default="analytics-processing-queue") + # Caps how many activities the worker runs simultaneously, regardless of how + # many workflows are started/queued — this is what actually protects the DB + # pool from an unpredictable event burst. Without a cap, every incoming + # event immediately becomes a concurrent DB-touching activity (load-tested: + # a 500-event burst instantly saturated a 100-connection Postgres instance). + # Anything beyond this limit waits safely in Temporal's own task queue + # instead of piling onto Postgres. Keep at or below DATABASE_POOL_MAX_SIZE. + WORKER_MAX_CONCURRENT_ACTIVITIES: int = Field(default=40, gt=0) + + # Image processing (deface_blur_activity) concurrency — tune these against + # actual server specs (CPU cores, available memory), not whatever machine + # they were load-tested on. Download/upload are cheap I/O; face-blur spawns + # a real subprocess with a fresh ONNX model load every call and is the + # memory-expensive step — see deface_blur_activity.py for why these are + # treated as separate concerns rather than one concurrency number. + IMAGE_EXECUTOR_MAX_WORKERS: int = Field(default_factory=lambda: max(4, (os.cpu_count() or 4) * 2), gt=0) + PER_SUBMISSION_IMAGE_CONCURRENCY: int = Field(default=3, gt=0) + BLUR_CONCURRENCY_LIMIT: int = Field(default=2, gt=0) # API Authentication — single shared Bearer token, checked via # secrets.compare_digest in app/api/deps.py. Required (no default): the app From fef46e148407fbe19a56bcd6d8f80e29c434db9f Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:07:33 +0530 Subject: [PATCH 10/10] fix(review): address CodeRabbit findings on PR #7 - config.py: reject DATABASE_POOL_MIN_SIZE > DATABASE_POOL_MAX_SIZE at settings-load time instead of failing later with a generic asyncpg error - deface_blur_activity.py: validate resolved image host against MEDIA_BASE_URL before downloading (SSRF), and check BaseException instead of Exception in the gather() results loop so CancelledError propagates correctly instead of raising a confusing TypeError - docker-compose.yaml: parameterize the hardcoded postgres:postgres credentials via env var substitution (defaults unchanged) - pytest.ini: add explicit pythonpath so imports don't depend on invocation directory/style - fix 4 kafka event fixtures (3 discussion + 1 story) that were missing required ingestion fields (pdfUrls, transcriptLink, solutions/challenges, participantsData) and would have been DLQ'd before reaching the classification scenarios they were meant to test - tests/unit_testing.py: remove a real network call firing before its mock patch in test_rating_002, and a dead if-False conditional in test_upload_004 --- app/config.py | 14 ++++++++- app/temporal/deface_blur_activity.py | 29 ++++++++++++++++++- docker-compose.yaml | 8 ++--- pytest.ini | 1 + ...eate_discussion_multi_statement_array.json | 18 +++++++++--- .../create_discussion_multi_theme_llm.json | 15 +++++++--- .../create_discussion_multi_theme_local.json | 16 +++++++--- ...eate_story_multi_barrier_single_theme.json | 24 ++++++++++----- tests/unit_testing.py | 13 ++------- 9 files changed, 101 insertions(+), 37 deletions(-) diff --git a/app/config.py b/app/config.py index 0150b0b..8910df7 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,7 @@ import json import os from typing import Dict, Any, List -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): @@ -156,6 +156,18 @@ def validate_database_url(cls, v: str) -> str: return v.replace("postgresql+asyncpg://", "postgresql://") return v + @model_validator(mode="after") + def validate_pool_bounds(self) -> "Settings": + # Without this, an inconsistent env config loads without error and only + # fails later at asyncpg.create_pool() with a generic ValueError that + # doesn't name the misconfigured variables. + if self.DATABASE_POOL_MIN_SIZE > self.DATABASE_POOL_MAX_SIZE: + raise ValueError( + f"DATABASE_POOL_MIN_SIZE ({self.DATABASE_POOL_MIN_SIZE}) must not exceed " + f"DATABASE_POOL_MAX_SIZE ({self.DATABASE_POOL_MAX_SIZE})." + ) + return self + @field_validator("PROCESS_CONFIG_STORY", "PROCESS_CONFIG_DISCUSSION") @classmethod def validate_process_config_json(cls, v: str, info) -> str: diff --git a/app/temporal/deface_blur_activity.py b/app/temporal/deface_blur_activity.py index 1e8d479..bcffadc 100644 --- a/app/temporal/deface_blur_activity.py +++ b/app/temporal/deface_blur_activity.py @@ -66,6 +66,21 @@ async def _run_in_image_executor(func, *args, **kwargs): return await loop.run_in_executor(_IMAGE_EXECUTOR, functools.partial(func, *args, **kwargs)) +def _is_allowed_media_host(resolved_url: str) -> bool: + """ + Restricts image downloads to MEDIA_BASE_URL's own host. image_urls come + from stored submission payloads — an absolute URL there was previously + passed straight to urlopen() with no host check, so a malicious or + compromised upstream could make the worker fetch internal endpoints + (SSRF), and the concurrent per-image fan-out would only increase how many + such requests could be issued at once. + """ + allowed_host = urllib.parse.urlparse(settings.MEDIA_BASE_URL).netloc.lower() + if not allowed_host: + return False + return urllib.parse.urlparse(resolved_url).netloc.lower() == allowed_host + + def _download_file(url: str, filename: str) -> Path: DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True) local_path = DOWNLOADS_DIR / filename @@ -93,6 +108,12 @@ async def _process_one_image(submission_id: str, tenant_code: str, sub_type: str else: resolved_url = url_str + if not _is_allowed_media_host(resolved_url): + raise ValueError( + f"Refusing to download image from disallowed host: {resolved_url!r} " + f"(only MEDIA_BASE_URL's host, {settings.MEDIA_BASE_URL!r}, is permitted)" + ) + parsed_path = urllib.parse.urlparse(resolved_url).path parts = [p for p in parsed_path.split("/") if p] if len(parts) >= 2: @@ -188,7 +209,13 @@ async def _bounded(i: int, url: Any) -> Dict[str, Any]: return_exceptions=True, ) for r in results: - if isinstance(r, Exception): + # BaseException, not Exception — return_exceptions=True also captures + # asyncio.CancelledError, which derives from BaseException. Temporal + # cancels activity coroutines on cancellation requests and heartbeat + # timeouts, so this path is reachable; missing it here would let a + # CancelledError reach the dict-indexing below and raise a confusing + # TypeError instead of properly propagating the cancellation. + if isinstance(r, BaseException): raise r blurred_local_paths = [r["public_url"] for r in results] diff --git a/docker-compose.yaml b/docker-compose.yaml index 71b9282..966f0c6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -72,7 +72,7 @@ services: image: elevate-analytics:latest env_file: .env environment: - DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + DATABASE_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@host.docker.internal:5432/analytics_db TEMPORAL_HOST: temporal:7233 KAFKA_BOOTSTRAP_SERVERS: kafka:9092 extra_hosts: @@ -94,7 +94,7 @@ services: image: elevate-analytics:latest env_file: .env environment: - DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + DATABASE_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@host.docker.internal:5432/analytics_db TEMPORAL_HOST: temporal:7233 KAFKA_BOOTSTRAP_SERVERS: kafka:9092 extra_hosts: @@ -114,7 +114,7 @@ services: image: elevate-analytics:latest env_file: .env environment: - DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + DATABASE_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@host.docker.internal:5432/analytics_db TEMPORAL_HOST: temporal:7233 KAFKA_BOOTSTRAP_SERVERS: kafka:9092 extra_hosts: @@ -137,7 +137,7 @@ services: image: elevate-analytics:latest env_file: .env environment: - DATABASE_URL: postgresql://postgres:postgres@host.docker.internal:5432/analytics_db + DATABASE_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@host.docker.internal:5432/analytics_db TEMPORAL_HOST: temporal:7233 KAFKA_BOOTSTRAP_SERVERS: kafka:9092 extra_hosts: diff --git a/pytest.ini b/pytest.ini index 19ef46d..8cecb5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] python_files = test_*.py unit_testing.py +pythonpath = . tests diff --git a/tests/kafka_events/create/create_discussion_multi_statement_array.json b/tests/kafka_events/create/create_discussion_multi_statement_array.json index 40ee414..26e8e48 100644 --- a/tests/kafka_events/create/create_discussion_multi_statement_array.json +++ b/tests/kafka_events/create/create_discussion_multi_statement_array.json @@ -21,15 +21,25 @@ "designation": "Women leader", "submissionDate": "2026-07-16T10:15:00Z", "imageUrls": [], - "pdfUrls": null, - "transcriptLink": null, + "pdfUrls": { + "original": "https://mohini-static.shikshalokam.org/chatbot/storymedia/9004/original.pdf", + "masked": "https://mohini-static.shikshalokam.org/chatbot/storymedia/masked/9004/original.pdf" + }, + "transcriptLink": "test9004transcriptlink0001", "challenges": [ "Our school children are not given books on time.", "There is a shortage of teachers in the school, due to which subject-wise studies are not done.", "The school is very far from the village." ], - "solutions": [], - "participantsData": [], + "solutions": [ + "Ensuring books are distributed to students at the start of the academic year.", + "Recruiting additional subject teachers to cover the shortage.", + "Arranging safe transportation for students from distant villages." + ], + "participantsData": [ + { "role": "men", "count": 3 }, + { "role": "women", "count": 5 } + ], "author": "9004", "language": "en" } diff --git a/tests/kafka_events/create/create_discussion_multi_theme_llm.json b/tests/kafka_events/create/create_discussion_multi_theme_llm.json index 1566cf7..9c80bf4 100644 --- a/tests/kafka_events/create/create_discussion_multi_theme_llm.json +++ b/tests/kafka_events/create/create_discussion_multi_theme_llm.json @@ -21,13 +21,20 @@ "designation": "Women leader", "submissionDate": "2026-07-16T10:05:00Z", "imageUrls": [], - "pdfUrls": null, - "transcriptLink": null, + "pdfUrls": { + "original": "https://mohini-static.shikshalokam.org/chatbot/storymedia/9002/original.pdf", + "masked": "https://mohini-static.shikshalokam.org/chatbot/storymedia/masked/9002/original.pdf" + }, + "transcriptLink": "test9002transcriptlink0001", "challenges": [ "Since her father comes home intoxicated most nights and stays out late, the girl feels too frightened to walk back home from school alone once evening falls." ], - "solutions": [], - "participantsData": [], + "solutions": [ + "Organizing a community escort/buddy system so girls are accompanied home safely in the evenings." + ], + "participantsData": [ + { "role": "women", "count": 4 } + ], "author": "9002", "language": "en" } diff --git a/tests/kafka_events/create/create_discussion_multi_theme_local.json b/tests/kafka_events/create/create_discussion_multi_theme_local.json index ca0d001..4af005f 100644 --- a/tests/kafka_events/create/create_discussion_multi_theme_local.json +++ b/tests/kafka_events/create/create_discussion_multi_theme_local.json @@ -21,13 +21,21 @@ "designation": "Women leader", "submissionDate": "2026-07-16T10:00:00Z", "imageUrls": [], - "pdfUrls": null, - "transcriptLink": null, + "pdfUrls": { + "original": "https://mohini-static.shikshalokam.org/chatbot/storymedia/9001/original.pdf", + "masked": "https://mohini-static.shikshalokam.org/chatbot/storymedia/masked/9001/original.pdf" + }, + "transcriptLink": "test9001transcriptlink0001", "challenges": [ "There is no school in the village after middle school, so children have to go far away, and due to the negligence of parents, both parents and children do not want to study further." ], - "solutions": [], - "participantsData": [], + "solutions": [ + "Setting up a secondary school closer to the village and running awareness sessions for parents on the importance of continued education." + ], + "participantsData": [ + { "role": "men", "count": 2 }, + { "role": "women", "count": 3 } + ], "author": "9001", "language": "en" } diff --git a/tests/kafka_events/create/create_story_multi_barrier_single_theme.json b/tests/kafka_events/create/create_story_multi_barrier_single_theme.json index 328f2fc..6f79d53 100644 --- a/tests/kafka_events/create/create_story_multi_barrier_single_theme.json +++ b/tests/kafka_events/create/create_story_multi_barrier_single_theme.json @@ -21,14 +21,22 @@ "designation": "Facilitator", "submissionDate": "2026-07-16T10:10:00Z", "imageUrls": [], - "pdfUrls": null, - "transcriptLink": null, + "pdfUrls": { + "original": "https://mohini-static.shikshalokam.org/chatbot/storymedia/9003/original.pdf", + "masked": "https://mohini-static.shikshalokam.org/chatbot/storymedia/masked/9003/original.pdf" + }, + "transcriptLink": "test9003transcriptlink0001", "objective": "There is no school in the village after middle school, so children have to go far away, and due to the negligence of parents, both parents and children do not want to study further.", - "challenges": [], - "actionSteps": [], - "impact": "", - "duration": "", - "blurb": "", - "content": "" + "challenges": [ + "There is no school in the village after middle school, so children have to go far away, and due to the negligence of parents, both parents and children do not want to study further." + ], + "actionSteps": [ + "Advocated with the local education office to set up a secondary school closer to the village.", + "Ran awareness sessions with parents on the importance of continued education." + ], + "impact": "More children from the village continued their education beyond middle school.", + "duration": "3 months", + "blurb": "A facilitator worked with the community to address why children were dropping out after middle school.", + "content": "A facilitator in the village noticed that children were not continuing their education after middle school, both because the nearest secondary school was far away and because some parents were not prioritizing continued schooling. The facilitator advocated with the local education office to establish a closer secondary school and ran awareness sessions with parents about the importance of continued education. Over three months, more children from the village continued their education beyond middle school." } } diff --git a/tests/unit_testing.py b/tests/unit_testing.py index f44421d..8987eaf 100644 --- a/tests/unit_testing.py +++ b/tests/unit_testing.py @@ -1316,17 +1316,8 @@ async def fake_get_submission_type_and_payload(c, sid, tenant): def test_rating_002_pdf_failure_falls_back_to_fields(monkeypatch): async def run_test(): - content, source, total_chars = rating_module._fetch_story_content( - pdf_url="https://example.com/broken.pdf", - challenge="A challenge statement here", action_steps="Some action steps", - impact="Some impact", submission_id="1", tenant_code="mitra", log_prefix="[test]", - ) - - def fake_download_that_fails(url, local_path): - raise RuntimeError("404 not found") - monkeypatch.setattr(rating_module, "_download_file", MagicMock(side_effect=RuntimeError("404 not found"))) - content, source, total_chars = rating_module._fetch_story_content( + content, source, _total_chars = rating_module._fetch_story_content( pdf_url="https://example.com/broken.pdf", challenge="A challenge statement here", action_steps="Some action steps", impact="Some impact", submission_id="1", tenant_code="mitra", log_prefix="[test]", @@ -1739,7 +1730,7 @@ def test_upload_004_valid_discussion_csv_uploads_successfully(test_client, monke files=_csv_file("valid_discussion.csv"), ) assert resp.status_code == 200 - assert resp.json()["report_type"] if False else resp.json()["status"] == "pending" + assert resp.json()["status"] == "pending" def test_upload_005_invalid_report_type_rejected(test_client):