diff --git a/data-analytics/lambda_functions/volunteer_application_analytics.py b/data-analytics/lambda_functions/volunteer_application_analytics.py index 419aa26..1fa75c7 100644 --- a/data-analytics/lambda_functions/volunteer_application_analytics.py +++ b/data-analytics/lambda_functions/volunteer_application_analytics.py @@ -42,6 +42,54 @@ def parse_event_body(event): return {} +def build_date_filter_trend(event): + time_range = event.get("time_range", "All") + + if time_range == "7D": + return "vd.created_at >= CURRENT_DATE - INTERVAL '7 days'" + if time_range == "30D": + return "vd.created_at >= CURRENT_DATE - INTERVAL '30 days'" + if time_range == "1Y": + return "vd.created_at >= CURRENT_DATE - INTERVAL '1 year'" + if time_range == "Custom": + start_date = event.get("start_date") + end_date = event.get("end_date") + if start_date and end_date: + return f"vd.created_at BETWEEN '{start_date}' AND '{end_date}'" + return "" + + return "" + +def build_date_filter_location(event): + time_range = event.get("time_range_location", "All") + + if time_range == "7D": + return "vd.created_at >= CURRENT_DATE - INTERVAL '7 days'" + if time_range == "30D": + return "vd.created_at >= CURRENT_DATE - INTERVAL '30 days'" + if time_range == "1Y": + return "vd.created_at >= CURRENT_DATE - INTERVAL '1 year'" + if time_range == "Custom": + start_date = event.get("location_start_date") + end_date = event.get("location_end_date") + if start_date and end_date: + return f"vd.created_at BETWEEN '{start_date}' AND '{end_date}'" + return "" + + return "" + +def get_grouping(time_range): + if time_range in ("7D", "30D", "Custom"): + return ( + "DATE(vd.created_at)", + "TO_CHAR(DATE(vd.created_at), 'YYYY-MM-DD')" + ) + + return ( + "DATE_TRUNC('month', vd.created_at)", + "TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM')" + ) + def lambda_handler(event, context): conn_V = None cursor_V = None @@ -72,11 +120,11 @@ def lambda_handler(event, context): chart_type = request_body.get("chart_type", "Bar Chart") skill = request_body.get("skill", "All Skills") - volunteer_activity_trend_virginia = get_volunteer_activity_trend(cursor_V, REAL_TABLE_USERS_VIRGINIA, REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA) - volunteers_by_location_virginia = get_volunteers_by_location(cursor_V,REAL_TABLE_USERS_VIRGINIA,REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA,REAL_TABLE_COUNTRY_VIRGINIA,REAL_TABLE_USER_SKILL_VIRGINIA,REAL_TABLE_HELP_CATEGORIES_VIRGINIA,country,chart_type,skill) + volunteer_activity_trend_virginia = get_volunteer_activity_trend(cursor_V, REAL_TABLE_USERS_VIRGINIA, REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA, request_body) + volunteers_by_location_virginia = get_volunteers_by_location(cursor_V,REAL_TABLE_USERS_VIRGINIA,REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA,REAL_TABLE_COUNTRY_VIRGINIA,REAL_TABLE_USER_SKILL_VIRGINIA,REAL_TABLE_HELP_CATEGORIES_VIRGINIA,country,chart_type,skill,request_body) - volunteer_activity_trend_ireland = get_volunteer_activity_trend(cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND) - volunteers_by_location_ireland = get_volunteers_by_location(cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND, REAL_TABLE_COUNTRY_IRELAND, REAL_TABLE_USER_SKILLS_IRELAND, REAL_TABLE_HELP_CATEGORIES_IRELAND,country, chart_type,skill) + volunteer_activity_trend_ireland = get_volunteer_activity_trend(cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND, request_body) + volunteers_by_location_ireland = get_volunteers_by_location(cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND, REAL_TABLE_COUNTRY_IRELAND, REAL_TABLE_USER_SKILLS_IRELAND, REAL_TABLE_HELP_CATEGORIES_IRELAND,country, chart_type,skill,request_body) response_data = { "volunteer_activity_trend" : merge_volunteer_activity_trend(volunteer_activity_trend_virginia, volunteer_activity_trend_ireland ), @@ -122,49 +170,55 @@ def lambda_handler(event, context): -def get_volunteer_activity_trend(cursor,users,volunteer_details): - try: - query1 = f"""SELECT TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM') AS month, - COUNT(DISTINCT u.user_id) AS count - FROM {users} u - JOIN {volunteer_details} vd ON u.user_id = vd.user_id - WHERE vd.created_at IS NOT NULL - GROUP BY 1 +def get_volunteer_activity_trend(cursor,users,volunteer_details,event=None): + try: + event = event or {} + time_range = event.get("time_range", "All") + date_filter = build_date_filter_trend(event) + group_by, period_select = get_grouping(time_range) + date_condition = f" AND {date_filter}" if date_filter else "" + + query1 = f"""SELECT {period_select} AS period, + COUNT(DISTINCT u.user_id) AS count + FROM {users} u + JOIN {volunteer_details} vd ON u.user_id = vd.user_id + WHERE vd.created_at IS NOT NULL{date_condition} + GROUP BY {group_by} ORDER BY 1 ASC""" cursor.execute(query1) - + new_volunteers = cursor.fetchall() - new_volunteers_final = [{"month": row[0], "count": int(row[1])} for row in new_volunteers] + new_volunteers_final = [{"period": row[0], "count": int(row[1])} for row in new_volunteers] - query2 = f""" - SELECT TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM') AS month, - COUNT(DISTINCT u.user_id) AS count FROM {users} u + query2 = f""" + SELECT {period_select} AS period, + COUNT(DISTINCT u.user_id) AS count FROM {users} u JOIN {volunteer_details} vd ON u.user_id = vd.user_id - WHERE vd.created_at IS NOT NULL - AND u.user_status_id = 1 - GROUP BY 1 + WHERE vd.created_at IS NOT NULL{date_condition} + AND u.user_status_id = 1 + GROUP BY {group_by} ORDER BY 1 ASC """ cursor.execute(query2) active_volunteers = cursor.fetchall() - active_volunteers_final = [{"month": row[0], "count": int(row[1])} for row in active_volunteers] + active_volunteers_final = [{"period": row[0], "count": int(row[1])} for row in active_volunteers] query3 = f""" - SELECT month, SUM(count) OVER (ORDER BY month) AS count - FROM ( SELECT TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM') AS month, + SELECT period, SUM(count) OVER (ORDER BY period) AS count + FROM ( SELECT {period_select} AS period, COUNT(DISTINCT u.user_id) AS count FROM {users} u JOIN {volunteer_details} vd ON u.user_id = vd.user_id - WHERE vd.created_at IS NOT NULL - GROUP BY 1 ) sub - ORDER BY month ASC; + WHERE vd.created_at IS NOT NULL{date_condition} + GROUP BY {group_by} ) sub + ORDER BY period ASC; """ - + cursor.execute(query3) total_volunteers = cursor.fetchall() - total_volunteers_final = [{"month": row[0], "count": int(row[1])} for row in total_volunteers] + total_volunteers_final = [{"period": row[0], "count": int(row[1])} for row in total_volunteers] return { "new_volunteers": new_volunteers_final, "active_volunteers": active_volunteers_final, @@ -181,15 +235,15 @@ def get_volunteer_activity_trend(cursor,users,volunteer_details): def merge_monthly_data(list1, list2): merged= {} - for row in list1 + list2 : - month = row['month'] + for row in list1 + list2 : + period = row['period'] count = row['count'] - merged[month] = merged.get(month,0) + count + merged[period] = merged.get(period,0) + count return [ - {'month': month, 'count': merged[month]} - for month in sorted(merged.keys())] + {'period': period, 'count': merged[period]} + for period in sorted(merged.keys())] def merge_volunteer_activity_trend(volunteer_activity_trend_virginia, volunteer_activity_trend_ireland): return { @@ -215,8 +269,8 @@ def merge_volunteer_by_location(list1, list2): return [ {"country": country, "count": merged[country]} for country in sorted(merged.keys()) ] -def get_volunteers_by_location( cursor, users, volunteer_details, country_table, user_skills,help_categories,country='All Countries',chart_type="Bar Chart",skill="All Skills"): - try: +def get_volunteers_by_location( cursor, users, volunteer_details, country_table, user_skills,help_categories,country='All Countries',chart_type="Bar Chart",skill="All Skills",event=None): + try: query= f"""SELECT COALESCE(c.country_code, 'Unknown') AS country, COUNT(DISTINCT u.user_id) AS count @@ -227,10 +281,13 @@ def get_volunteers_by_location( cursor, users, volunteer_details, country_table, ON u.country_id = c.country_id WHERE 1=1 """ - + params = [] - + date_filter = build_date_filter_location(event or {}) + if date_filter: + query += f" AND {date_filter}" + if country != "All Countries": query += " AND UPPER(c.country_code) = %s" params.append(country) @@ -297,9 +354,100 @@ def get_db_config(db): if __name__ == "__main__": - test_event = {} - print(lambda_handler(test_event, None)) - + import csv + import os + + SQL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "sql") + + # Local database only -- connection details come from environment + # variables with localhost defaults, never from SSM. + LOCAL_DB_CONFIG = { + "host": os.environ.get("LOCAL_DB_HOST", "localhost"), + "port": int(os.environ.get("LOCAL_DB_PORT", "5432")), + "dbname": os.environ.get("LOCAL_DB_NAME", "postgres"), + "user": os.environ.get("LOCAL_DB_USER", "postgres"), + "password": os.environ.get("LOCAL_DB_PASSWORD", "postgres"), + } + + def read_csv(filename): + path = os.path.join(SQL_DIR, filename) + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + def clean(value): + if value in (None, "", "NULL"): + return None + return value + + def load_local_test_data(conn): + users_rows = read_csv("users.csv") + volunteer_rows = read_csv("volunteer_details.csv") + country_rows = read_csv("country.csv") + skill_rows = read_csv("user_skills.csv") + category_rows = read_csv("help_category.csv") + + cur = conn.cursor() + for schema in ("virginia_dev_saayam_rdbms", "ireland_dev_saayam_rdbms"): + cur.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE") + cur.execute(f"CREATE SCHEMA {schema}") + cur.execute(f"CREATE TABLE {schema}.users (user_id TEXT, country_id INT, user_status_id INT)") + cur.execute(f"CREATE TABLE {schema}.volunteer_details (user_id TEXT, created_at TIMESTAMP)") + cur.execute(f"CREATE TABLE {schema}.country (country_id INT, country_code TEXT)") + cur.execute(f"CREATE TABLE {schema}.user_skills (user_id TEXT, cat_id TEXT)") + cur.execute(f"CREATE TABLE {schema}.help_categories (cat_id TEXT, cat_name TEXT)") + + # The CSV fixtures represent one region; load them into the + # Virginia schema and leave Ireland empty. + schema = "virginia_dev_saayam_rdbms" + cur.executemany( + f"INSERT INTO {schema}.users VALUES (%s, %s, %s)", + [(r["user_id"], clean(r["country_id"]), clean(r["user_status_id"])) for r in users_rows], + ) + cur.executemany( + f"INSERT INTO {schema}.volunteer_details VALUES (%s, %s)", + [(r["user_id"], clean(r["created_at"])) for r in volunteer_rows], + ) + cur.executemany( + f"INSERT INTO {schema}.country VALUES (%s, %s)", + [(clean(r["country_id"]), clean(r["country_code"])) for r in country_rows], + ) + cur.executemany( + f"INSERT INTO {schema}.user_skills VALUES (%s, %s)", + [(r["user_id"], r["cat_id"]) for r in skill_rows], + ) + cur.executemany( + f"INSERT INTO {schema}.help_categories VALUES (%s, %s)", + [(r["cat_id"], r["cat_name"]) for r in category_rows], + ) + conn.commit() + cur.close() + def local_db_config(db): + return LOCAL_DB_CONFIG + + try: + setup_conn = psycopg2.connect(**LOCAL_DB_CONFIG) + load_local_test_data(setup_conn) + setup_conn.close() + print("Local test data loaded from", SQL_DIR) + except Exception as e: + print("Could not prepare local test database:", str(e)) + raise SystemExit(1) + + get_db_config = local_db_config + + test_events = [ + {"time_range": "7D", "time_range_location": "7D"}, + {"time_range": "30D", "time_range_location": "30D"}, + {"time_range": "1Y", "time_range_location": "1Y"}, + {"time_range": "All", "time_range_location": "All"}, + {"time_range": "Custom", "start_date": "2026-01-01", "end_date": "2026-05-31", + "time_range_location": "Custom", "location_start_date": "2026-01-01", "location_end_date": "2026-05-31"}, + ] + + for test_event in test_events: + print("Event:", json.dumps(test_event)) + result = lambda_handler(test_event, None) + print("Response:", result["body"]) + print() -