Skip to content

Add date filter support for Volunteer Activity Trend and Volunteers by Location #217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mahaveerappini wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add date filter support for Volunteer Activity Trend and Volunteers by Location #217

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 174 additions & 59 deletions data-analytics/lambda_functions/volunteer_application_analytics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import os
import psycopg2
import boto3

Expand Down Expand Up @@ -71,12 +72,38 @@ def lambda_handler(event, context):
country = request_body.get("country", "All Countries")
chart_type = request_body.get("chart_type", "Bar Chart")
skill = request_body.get("skill", "All Skills")

volunteer_activity_trend_virginia = get_volunteer_activity_trend(cursor_V, REAL_TABLE_USERS_VIRGINIA, REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA)
volunteers_by_location_virginia = get_volunteers_by_location(cursor_V,REAL_TABLE_USERS_VIRGINIA,REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA,REAL_TABLE_COUNTRY_VIRGINIA,REAL_TABLE_USER_SKILL_VIRGINIA,REAL_TABLE_HELP_CATEGORIES_VIRGINIA,country,chart_type,skill)

volunteer_activity_trend_ireland = get_volunteer_activity_trend(cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND)
volunteers_by_location_ireland = get_volunteers_by_location(cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND, REAL_TABLE_COUNTRY_IRELAND, REAL_TABLE_USER_SKILLS_IRELAND, REAL_TABLE_HELP_CATEGORIES_IRELAND,country, chart_type,skill)
# Date filter for the Volunteer Activity Trend chart
time_range = request_body.get("time_range", "All")
start_date = request_body.get("start_date")
end_date = request_body.get("end_date")

# Independent date filter for the Volunteers by Location chart
time_range_location = request_body.get("time_range_location", "All")
location_start_date = request_body.get("location_start_date")
location_end_date = request_body.get("location_end_date")

volunteer_activity_trend_virginia = get_volunteer_activity_trend(
cursor_V, REAL_TABLE_USERS_VIRGINIA, REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA,
time_range, start_date, end_date
)
volunteers_by_location_virginia = get_volunteers_by_location(
cursor_V, REAL_TABLE_USERS_VIRGINIA, REAL_TABLE_VOLUNTEER_DETAILS_VIRGINIA,
REAL_TABLE_COUNTRY_VIRGINIA, REAL_TABLE_USER_SKILL_VIRGINIA, REAL_TABLE_HELP_CATEGORIES_VIRGINIA,
country, chart_type, skill,
time_range_location, location_start_date, location_end_date
)

volunteer_activity_trend_ireland = get_volunteer_activity_trend(
cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND,
time_range, start_date, end_date
)
volunteers_by_location_ireland = get_volunteers_by_location(
cursor_I, REAL_TABLE_USERS_IRELAND, REAL_TABLE_VOLUNTEER_DETAILS_IRELAND,
REAL_TABLE_COUNTRY_IRELAND, REAL_TABLE_USER_SKILLS_IRELAND, REAL_TABLE_HELP_CATEGORIES_IRELAND,
country, chart_type, skill,
time_range_location, location_start_date, location_end_date
)

response_data = {
"volunteer_activity_trend" : merge_volunteer_activity_trend(volunteer_activity_trend_virginia, volunteer_activity_trend_ireland ),
Expand All @@ -93,7 +120,7 @@ def lambda_handler(event, context):
},
"body": json.dumps(response_data)
}

except Exception as e:
# Catches an error if connection fails
print("ERROR:", str(e))
Expand All @@ -119,52 +146,119 @@ def lambda_handler(event, context):
if conn_I:
conn_I.close()
print("Ireland Database connection closed")



def get_volunteer_activity_trend(cursor,users,volunteer_details):
try:
query1 = f"""SELECT TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM') AS month,
COUNT(DISTINCT u.user_id) AS count
FROM {users} u
JOIN {volunteer_details} vd ON u.user_id = vd.user_id
WHERE vd.created_at IS NOT NULL
GROUP BY 1

SUPPORTED_TIME_RANGES = {"7D", "30D", "1Y", "All", "Custom"}


def _resolve_date_filter(time_range, start_date, end_date, column="vd.created_at"):
"""
Shared implementation backing build_date_filter_trend() and
build_date_filter_location(). Returns a tuple of
(sql_condition_string, params_list) meant to be spliced into a WHERE
clause. Never interpolates start_date/end_date directly into the SQL
string - they are always passed back as bind params.
"""
if time_range == "7D":
return f"{column} >= CURRENT_DATE - INTERVAL '7 days'", []
elif time_range == "30D":
return f"{column} >= CURRENT_DATE - INTERVAL '30 days'", []
elif time_range == "1Y":
return f"{column} >= CURRENT_DATE - INTERVAL '1 year'", []
elif time_range == "Custom":
if not start_date or not end_date:
# Missing custom bounds -> don't filter rather than error the whole widget out.
return "1=1", []
return f"{column} BETWEEN %s AND %s", [start_date, end_date]
else:
# "All" (or any unrecognized value) -> no date filtering.
return "1=1", []


def build_date_filter_trend(time_range, start_date=None, end_date=None, column="vd.created_at"):
"""
Date filter condition for the Volunteer Activity Trend chart.
Used in the WHERE clause of every query inside get_volunteer_activity_trend().
"""
return _resolve_date_filter(time_range, start_date, end_date, column)


def build_date_filter_location(time_range_location, location_start_date=None, location_end_date=None, column="vd.created_at"):
"""
Date filter condition for the Volunteers by Location chart.
Uses the same filtering logic as build_date_filter_trend(), but is kept
as its own function so the two charts' filters stay independent.
"""
return _resolve_date_filter(time_range_location, location_start_date, location_end_date, column)


def get_grouping(time_range):
"""
Determines how Volunteer Activity Trend results are bucketed:
7D, 30D, Custom -> "day"
1Y, All -> "month"
This value is passed straight into DATE_TRUNC().
"""
if time_range in ("7D", "30D", "Custom"):
return "day"
# "1Y", "All", and any unrecognized value default to monthly grouping.
return "month"


def _to_char_format_for_grouping(grouping):
return "YYYY-MM-DD" if grouping == "day" else "YYYY-MM"


def get_volunteer_activity_trend(cursor, users, volunteer_details, time_range="All", start_date=None, end_date=None):
try:
date_filter_sql, date_params = build_date_filter_trend(time_range, start_date, end_date)
trunc_unit = get_grouping(time_range)
date_format = _to_char_format_for_grouping(trunc_unit)

query1 = f"""SELECT TO_CHAR(DATE_TRUNC(%s, vd.created_at), %s) AS period,
COUNT(DISTINCT u.user_id) AS count
FROM {users} u
JOIN {volunteer_details} vd ON u.user_id = vd.user_id
WHERE vd.created_at IS NOT NULL
AND {date_filter_sql}
GROUP BY 1
ORDER BY 1 ASC"""
cursor.execute(query1)
cursor.execute(query1, [trunc_unit, date_format] + date_params)


new_volunteers = cursor.fetchall()
new_volunteers_final = [{"month": row[0], "count": int(row[1])} for row in new_volunteers]
new_volunteers_final = [{"period": row[0], "count": int(row[1])} for row in new_volunteers]

query2 = f"""
SELECT TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM') AS month,
COUNT(DISTINCT u.user_id) AS count FROM {users} u
query2 = f"""
SELECT TO_CHAR(DATE_TRUNC(%s, vd.created_at), %s) AS period,
COUNT(DISTINCT u.user_id) AS count FROM {users} u
JOIN {volunteer_details} vd ON u.user_id = vd.user_id
WHERE vd.created_at IS NOT NULL
AND u.user_status_id = 1
AND {date_filter_sql}
AND u.user_status_id = 1
GROUP BY 1
ORDER BY 1 ASC
"""
cursor.execute(query2)
cursor.execute(query2, [trunc_unit, date_format] + date_params)
active_volunteers = cursor.fetchall()
active_volunteers_final = [{"month": row[0], "count": int(row[1])} for row in active_volunteers]
active_volunteers_final = [{"period": row[0], "count": int(row[1])} for row in active_volunteers]

query3 = f"""
SELECT month, SUM(count) OVER (ORDER BY month) AS count
FROM ( SELECT TO_CHAR(DATE_TRUNC('month', vd.created_at), 'YYYY-MM') AS month,
SELECT period, SUM(count) OVER (ORDER BY period) AS count
FROM ( SELECT TO_CHAR(DATE_TRUNC(%s, vd.created_at), %s) AS period,
COUNT(DISTINCT u.user_id) AS count
FROM {users} u
JOIN {volunteer_details} vd
ON u.user_id = vd.user_id
WHERE vd.created_at IS NOT NULL
AND {date_filter_sql}
GROUP BY 1 ) sub
ORDER BY month ASC;
ORDER BY period ASC;
"""
cursor.execute(query3)

cursor.execute(query3, [trunc_unit, date_format] + date_params)
total_volunteers = cursor.fetchall()
total_volunteers_final = [{"month": row[0], "count": int(row[1])} for row in total_volunteers]
total_volunteers_final = [{"period": row[0], "count": int(row[1])} for row in total_volunteers]
return {
"new_volunteers": new_volunteers_final,
"active_volunteers": active_volunteers_final,
Expand All @@ -179,29 +273,29 @@ def get_volunteer_activity_trend(cursor,users,volunteer_details):
"total_volunteers": []
}

def merge_monthly_data(list1, list2):
merged= {}
for row in list1 + list2 :
month = row['month']
def merge_trend_data(list1, list2):
merged = {}
for row in list1 + list2:
period = row['period']
count = row['count']

merged[month] = merged.get(month,0) + count
merged[period] = merged.get(period, 0) + count

return [
{'month': month, 'count': merged[month]}
for month in sorted(merged.keys())]
{'period': period, 'count': merged[period]}
for period in sorted(merged.keys())]

def merge_volunteer_activity_trend(volunteer_activity_trend_virginia, volunteer_activity_trend_ireland):
return {
"new_volunteers": merge_monthly_data(
"new_volunteers": merge_trend_data(
volunteer_activity_trend_virginia.get("new_volunteers", []),
volunteer_activity_trend_ireland.get("new_volunteers", [])
),
"active_volunteers": merge_monthly_data(
"active_volunteers": merge_trend_data(
volunteer_activity_trend_virginia.get("active_volunteers", []),
volunteer_activity_trend_ireland.get("active_volunteers", [])
),
"total_volunteers": merge_monthly_data(
"total_volunteers": merge_trend_data(
volunteer_activity_trend_virginia.get("total_volunteers", []),
volunteer_activity_trend_ireland.get("total_volunteers", [])
)
Expand All @@ -215,8 +309,10 @@ def merge_volunteer_by_location(list1, list2):
return [ {"country": country, "count": merged[country]} for country in sorted(merged.keys()) ]


def get_volunteers_by_location( cursor, users, volunteer_details, country_table, user_skills,help_categories,country='All Countries',chart_type="Bar Chart",skill="All Skills"):
try:
def get_volunteers_by_location( cursor, users, volunteer_details, country_table, user_skills,help_categories,country='All Countries',chart_type="Bar Chart",skill="All Skills", time_range_location="All", location_start_date=None, location_end_date=None):
try:
date_filter_sql, date_params = build_date_filter_location(time_range_location, location_start_date, location_end_date)

query= f"""SELECT
COALESCE(c.country_code, 'Unknown') AS country,
COUNT(DISTINCT u.user_id) AS count
Expand All @@ -225,12 +321,12 @@ def get_volunteers_by_location( cursor, users, volunteer_details, country_table,
ON u.user_id = vd.user_id
LEFT JOIN {country_table} c
ON u.country_id = c.country_id
WHERE 1=1
WHERE {date_filter_sql}
"""

params = []


params = list(date_params)


if country != "All Countries":
query += " AND UPPER(c.country_code) = %s"
params.append(country)
Expand Down Expand Up @@ -264,15 +360,30 @@ def get_volunteers_by_location( cursor, users, volunteer_details, country_table,
return []

def get_db_config(db):
ssm = boto3.client("ssm", region_name="us-east-1")

if db == "Virginia":
parameter_name = "/dev/saayam/db/Virginia/Analytics/user"
elif db == "Ireland":
parameter_name = "/dev/saayam/db/Ireland/Analytics/user"
else:
if db not in ("Virginia", "Ireland"):
raise ValueError("Database must be either Virginia or Ireland")

# Local testing path: set LOCAL_DB=true and supply connection details via
# environment variables (e.g. a .env file that is NOT committed). No
# production credentials are ever hardcoded here.
if os.environ.get("LOCAL_DB", "false").lower() == "true":
prefix = db.upper() # VIRGINIA or IRELAND
return {
"host": os.environ.get(f"{prefix}_DB_HOST", os.environ.get("LOCAL_DB_HOST", "localhost")),
"port": int(os.environ.get(f"{prefix}_DB_PORT", os.environ.get("LOCAL_DB_PORT", "5432"))),
"dbname": os.environ.get(f"{prefix}_DB_NAME", os.environ.get("LOCAL_DB_NAME", "saayam")),
"user": os.environ.get(f"{prefix}_DB_USER", os.environ.get("LOCAL_DB_USER", "postgres")),
"password": os.environ.get(f"{prefix}_DB_PASSWORD", os.environ.get("LOCAL_DB_PASSWORD", "")),
}

# Production/dev path: pull credentials from AWS SSM Parameter Store.
ssm = boto3.client("ssm", region_name="us-east-1")
parameter_name = (
"/dev/saayam/db/Virginia/Analytics/user"
if db == "Virginia"
else "/dev/saayam/db/Ireland/Analytics/user"
)

response = ssm.get_parameter(
Name=parameter_name,
WithDecryption=True
Expand All @@ -297,9 +408,13 @@ def get_db_config(db):


if __name__ == "__main__":
test_event = {}
print(lambda_handler(test_event, None))




# Local run example: set LOCAL_DB=true (plus *_DB_HOST/PORT/NAME/USER/PASSWORD
# env vars pointing at a local Postgres loaded with the sql/ fixtures) before
# running `python volunteer_application_analytics.py`.
test_event = {
"body": json.dumps({
"time_range": "30D",
"time_range_location": "7D"
})
}
print(lambda_handler(test_event, None))