From a8515a89c64ce016e30e87d8b617a54313d0623d Mon Sep 17 00:00:00 2001 From: Graham Hukill Date: Thu, 4 Sep 2025 10:33:43 -0400 Subject: [PATCH] Refine logic for Employee Salary History unique key Why these changes are being introduced: A recently added integrity alert triggered for the 'Employee Salary History' table, indicating we had a row in Quickbase that no longer aligned with data warehouse data. As has happened before, it appears to be shifting data warehouse values, this time for "End Date" and reason and type data, that modifies the unique key. When the unique key changes, we get stale data in Quickbase because we are adding rows instead of updating them. How this addresses that need: After some analysis, and discussion with HR, it was determined we could remove some fields from the unique key logic which keeps it consistent between data warehouse changes. Part of this work was also skipping any salary history with a reason of "Lump Sum" which has only showed up once, and is the only time we might get multiple values for a single person, on a single day, for a single appointment. Similarly, we already skip "Supplemental Pay" for this reason. Side effects of this change: * Improve merge field values, reduce stale data in Salary History table. Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/IN-1450 --- hrqb/tasks/employee_salary_history.py | 2 -- hrqb/tasks/sql/employee_salary_history.sql | 2 ++ tests/tasks/test_employee_salary_history.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hrqb/tasks/employee_salary_history.py b/hrqb/tasks/employee_salary_history.py index 23b4013..58a9152 100644 --- a/hrqb/tasks/employee_salary_history.py +++ b/hrqb/tasks/employee_salary_history.py @@ -82,8 +82,6 @@ def get_dataframe(self) -> pd.DataFrame: row.mit_id, row.position_id, str(row.appointment_begin_date), - str(row.hr_personnel_action), - str(row.hr_action_reason), str(row.start_date), ] ), diff --git a/hrqb/tasks/sql/employee_salary_history.sql b/hrqb/tasks/sql/employee_salary_history.sql index 45e12eb..044f0ad 100644 --- a/hrqb/tasks/sql/employee_salary_history.sql +++ b/hrqb/tasks/sql/employee_salary_history.sql @@ -11,6 +11,7 @@ CHANGELOG - 2024-07-24 Added appointment begin/end date to help match appointments - 2024-09-18 Omit any HR_PERSONNEL_ACTION_TYPE rows where type is "Salary Supplement" - 2025-02-05 Remove 2019-01-01 date cutoff entirely + - 2025-09-04 Omit any HR_ACTION_REASON rows where reason is "Lump Sum" */ select distinct @@ -47,4 +48,5 @@ left join HR_PERSONNEL_ACTION_TYPE at on at.HR_PERSONNEL_ACTION_TYPE_KEY = a.HR_ left join HR_JOB j on j.HR_JOB_KEY = a.HR_JOB_KEY left join HR_POSITION p on p.HR_POSITION_KEY = a.HR_POSITION_KEY where at.HR_PERSONNEL_ACTION not in ('Salary Supplement') +and at.HR_ACTION_REASON not in ('Lump Sum') order by a.MIT_ID, a.APPT_TX_BEGIN_DATE, a.APPT_TX_END_DATE diff --git a/tests/tasks/test_employee_salary_history.py b/tests/tasks/test_employee_salary_history.py index 1441247..f4d5946 100644 --- a/tests/tasks/test_employee_salary_history.py +++ b/tests/tasks/test_employee_salary_history.py @@ -58,8 +58,6 @@ def test_task_transform_employee_salary_history_key_expected_from_input_data( emp_salary_row["MIT ID"], qb_emp_appt_row["Position ID"], qb_emp_appt_row["Begin Date"], - emp_salary_row["Related Salary Change Type"], - emp_salary_row["Salary Change Reason"], emp_salary_row["Start Date"], ] )