added project task and status report

manual_report_scripts/config.sample

@@ -0,0 +1,260 @@
+{
+  "druid_url": "http://localhost:8082",
+  "pairs": [
+    {
+      "program_id": "605083ba09b7bd61555580fb",
+      "solution_id": "5ff9dc1b9259097d48017bbe"
+    },
+    {
+      "program_id": "62205480a81abe61c07e5058",
+      "solution_id": "62206042a81abe61c07e505b"
+    }
+  ],
+  "ml-task-detail-exhaust": {
+    "id": "ml-task-detail-exhaust",
+    "labels": {
+      "block_name": "Block",
+      "board_name": "Declared Board",
+      "project_title_editable": "Project Title",
+      "task_evidence": "Task Evidence",
+      "project_completed_date": "Project completion date of the user",
+      "designation": "User sub type",
+      "project_created_date": "Project start date of the user",
+      "project_duration": "Project Duration",
+      "project_evidence": "Project Evidence",
+      "status_of_project": "Project Status",
+      "sub_task": "Sub-Tasks",
+      "tasks": "Tasks",
+      "project_id": "Project ID",
+      "project_description": "Project Objective",
+      "user_type": "User Type",
+      "program_externalId": "Program ID",
+      "organisation_name": "Org Name",
+      "createdBy": "UUID",
+      "area_of_improvement": "Category",
+      "school_name": "School Name",
+      "district_name": "District",
+      "program_name": "Program Name",
+      "project_remarks": "Project Remarks",
+      "school_code": "School ID",
+      "state_name": "Declared State",
+      "task_remarks": "Task Remarks"
+    },
+    "quoteColumns": [
+      "Project Title",
+      "Project Objective",
+      "Category",
+      "Tasks",
+      "Task Remarks",
+      "Sub-Tasks",
+      "Project Remarks"
+    ],
+    "dateRange": {
+      "interval": {
+        "startDate": "1901-01-01",
+        "endDate": "2101-01-01"
+      },
+      "granularity": "all",
+      "intervalSlider": 0
+    },
+    "metrics": [
+      {
+        "metric": "total_content_plays_on_portal",
+        "label": "total_content_plays_on_portal",
+        "druidQuery": {
+          "intervals": "1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00",
+          "dataSource": "sl-project",
+          "columns": [
+            "createdBy",
+            "user_type",
+            "designation",
+            "state_name",
+            "district_name",
+            "block_name",
+            "school_name",
+            "school_code",
+            "board_name",
+            "organisation_name",
+            "program_name",
+            "program_externalId",
+            "project_id",
+            "project_title_editable",
+            "project_description",
+            "area_of_improvement",
+            "project_created_date",
+            "project_completed_date",
+            "project_duration",
+            "status_of_project",
+            "tasks",
+            "sub_task",
+            "task_evidence",
+            "task_remarks",
+            "project_evidence",
+            "project_remarks"
+          ],
+          "queryType": "scan"
+        }
+      }
+    ],
+    "output": [
+      {
+        "zip": false,
+        "label": "",
+        "dims": [
+          "date"
+        ],
+        "fileParameters": [
+          "id",
+          "dims"
+        ],
+        "metrics": [
+          "createdBy",
+          "user_type",
+          "designation",
+          "state_name",
+          "district_name",
+          "block_name",
+          "school_name",
+          "school_code",
+          "board_name",
+          "organisation_name",
+          "program_name",
+          "program_externalId",
+          "project_id",
+          "project_title_editable",
+          "project_description",
+          "area_of_improvement",
+          "project_created_date",
+          "project_completed_date",
+          "project_duration",
+          "status_of_project",
+          "tasks",
+          "sub_task",
+          "task_evidence",
+          "task_remarks",
+          "project_evidence",
+          "project_remarks"
+        ],
+        "type": "csv"
+      }
+    ],
+    "sort": [
+      "UUID",
+      "Program ID",
+      "Project ID",
+      "Tasks"
+    ],
+    "queryType": "scan"
+  },
+  "ml-project-status-exhaust": {
+    "id": "ml-project-status-exhaust",
+    "labels": {
+      "block_name": "Block",
+      "board_name": "Declared Board",
+      "project_title_editable": "Project Title",
+      "project_completed_date": "Project completion date of the user",
+      "designation": "User sub type",
+      "project_created_date": "Project start date of the user",
+      "project_last_sync": "Project last Synced date",
+      "project_duration": "Project Duration",
+      "status_of_project": "Project Status",
+      "project_id": "Project ID",
+      "project_description": "Project Objective",
+      "user_type": "User Type",
+      "program_externalId": "Program ID",
+      "organisation_name": "Org Name",
+      "certificate_status_customised": "Certificate Status",
+      "createdBy": "UUID",
+      "school_name": "School Name",
+      "district_name": "District",
+      "program_name": "Program Name",
+      "school_code": "School ID",
+      "state_name": "Declared State"
+    },
+    "dateRange": {
+      "interval": {
+        "startDate": "1901-01-01",
+        "endDate": "2101-01-01"
+      },
+      "granularity": "all",
+      "intervalSlider": 0
+    },
+    "metrics": [
+      {
+        "metric": "total_content_plays_on_portal",
+        "label": "total_content_plays_on_portal",
+        "druidQuery": {
+          "intervals": "1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00",
+          "dataSource": "sl-project",
+          "columns": [
+            "createdBy",
+            "user_type",
+            "designation",
+            "state_name",
+            "district_name",
+            "block_name",
+            "school_name",
+            "school_code",
+            "board_name",
+            "organisation_name",
+            "program_name",
+            "program_externalId",
+            "project_id",
+            "project_title_editable",
+            "project_description",
+            "project_created_date",
+            "project_completed_date",
+            "project_duration",
+            "project_last_sync",
+            "status_of_project",
+            "certificate_status_customised"
+          ],
+          "queryType": "scan"
+        }
+      }
+    ],
+    "output": [
+      {
+        "zip": false,
+        "label": "",
+        "dims": [
+          "date"
+        ],
+        "fileParameters": [
+          "id",
+          "dims"
+        ],
+        "metrics": [
+          "createdBy",
+          "user_type",
+          "designation",
+          "state_name",
+          "district_name",
+          "block_name",
+          "school_name",
+          "school_code",
+          "board_name",
+          "organisation_name",
+          "program_name",
+          "program_externalId",
+          "project_id",
+          "project_title_editable",
+          "project_description",
+          "project_created_date",
+          "project_completed_date",
+          "project_duration",
+          "project_last_sync",
+          "status_of_project",
+          "certificate_status_customised"
+        ],
+        "type": "csv"
+      }
+    ],
+    "sort": [
+      "UUID",
+      "Program ID",
+      "Project ID"
+    ],
+    "queryType": "scan"
+  }
+}

manual_report_scripts/project_task_and_status_report_exhaust.py

@@ -0,0 +1,136 @@
+import json
+import csv
+import sys
+import requests
+import argparse
+from datetime import datetime
+
+def execute_report(report_name, report_config, expected_headers, out_prefix, druid_url, pairs, dedup_uuid, additional_filters=None):
+    print(f"\n--- Generating {report_name} ---")
+    query_url = f"{druid_url}/druid/v2/?pretty"
+
+    metrics = report_config.get("metrics", [])
+    if not metrics or "druidQuery" not in metrics[0]:
+        print(f"Skipping {report_name}: Missing metrics[0].druidQuery in config")
+        return
+
+    base_query = metrics[0]["druidQuery"]
+    labels_map = report_config.get("labels", {})
+    output_metrics = report_config.get("output", [{}])[0].get("metrics", [])
+
+    druid_columns = [m for m in output_metrics]
+
+    for pair in pairs:
+        program_id = pair.get("program_id")
+        solution_id = pair.get("solution_id")
+        if not program_id or not solution_id:
+            continue
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        out_file = f"{out_prefix}_{solution_id}_{timestamp}.csv"
+
+        with open(out_file, "w", newline="", encoding="utf-8") as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=expected_headers)
+            writer.writeheader()
+
+            seen_uuids = set()
+
+            payload = json.loads(json.dumps(base_query))
+            if "filter" not in payload:
+                payload["filter"] = {"type": "and", "fields": []}
+            elif payload["filter"]["type"] != "and":
+                payload["filter"] = {"type": "and", "fields": [payload["filter"]]}
+
+            payload["filter"]["fields"].extend([
+                {"type": "selector", "dimension": "program_id", "value": program_id},
+                {"type": "selector", "dimension": "solution_id", "value": solution_id}
+            ])
+            if additional_filters:
+                payload["filter"]["fields"].extend(additional_filters)
+
+            print(f"Querying {report_name} for program_id={program_id}, solution_id={solution_id} ...")
+            try:
+                response = requests.post(query_url, json=payload)
+                response.raise_for_status()
+            except Exception as e:
+                print(f"Error executing Druid query: {e}")
+                continue
+
+            results = response.json()
+            if not results:
+                continue
+
+            for res in results:
+                events = res.get("events", [])
+                for event in events:
+                    row = {}
+                    actual_event = event.get("event") if isinstance(event, dict) and "event" in event else event
+
+                    if dedup_uuid:
+                        uuid = str(actual_event.get("createdBy", "")).strip()
+                        if uuid in seen_uuids:
+                            continue
+                        seen_uuids.add(uuid)
+
+                    for d_col in druid_columns:
+                        header = labels_map.get(d_col, d_col)
+                        val = actual_event.get(d_col, "")
+                        if isinstance(val, list):
+                            val = ", ".join([str(v) for v in val])
+
+                        if header in expected_headers:
+                            row[header] = val
+
+                    writer.writerow(row)
+
+        print(f"Successfully generated {out_file}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Merged project task and status report using Druid")
+    parser.add_argument("--config", default="config.ini", help="Path to JSON config file")
+    parser.add_argument("--out_task_prefix", default="task_report", help="Prefix for output Task CSV")
+    parser.add_argument("--out_status_prefix", default="status_report", help="Prefix for output Status CSV")
+
+    args = parser.parse_args()
+
+    try:
+        with open(args.config, 'r') as f:
+            config = json.load(f)
+    except Exception as e:
+        print(f"Error loading config {args.config}: {e}")
+        sys.exit(1)
+
+    druid_url = config.get("druid_url", "http://localhost:8082")
+    pairs = config.get("pairs", [])
+    if not pairs:
+        print("No program/solution pairs provided in config.")
+        return
+
+    task_config = config.get("ml-task-detail-exhaust")
+    if task_config:
+        # Expected Headers constructed securely from the explicit map in the task JSON
+        expected_headers_task = [
+            "UUID", "User Type", "User sub type", "Declared State", "District", 
+            "Block", "School Name", "School ID", "Declared Board", "Org Name", 
+            "Program Name", "Program ID", "Project ID", "Project Title", "Project Objective", 
+            "Category", "Project start date of the user", "Project completion date of the user", 
+            "Project Duration", "Project Status", 
+            "Tasks", "Sub-Tasks", "Task Evidence", "Task Remarks", "Project Evidence", "Project Remarks"
+        ]
+        execute_report("Task Detail Exhaust", task_config, expected_headers_task, args.out_task_prefix, druid_url, pairs, dedup_uuid=False, additional_filters=[{"type": "selector", "dimension": "status_of_project", "value": "submitted"}])
+
+    status_config = config.get("ml-project-status-exhaust")
+    if status_config:
+        # Expected Headers constructed securely from the explicit map in the status JSON
+        expected_headers_status = [
+            "UUID", "User Type", "User sub type", "Declared State", "District", 
+            "Block", "School Name", "School ID", "Declared Board", "Org Name", 
+            "Program Name", "Program ID", "Project ID", "Project Title", "Project Objective", 
+            "Project start date of the user", "Project completion date of the user", 
+            "Project Duration", "Project last Synced date", "Project Status", 
+            "Certificate Status"
+        ]
+        execute_report("Project Status Exhaust", status_config, expected_headers_status, args.out_status_prefix, druid_url, pairs, dedup_uuid=True)
+
+if __name__ == "__main__":
+    main()