cleaned clean_salary method so that it converts all annual to hr

Author: dmorton714

src/notebooks/mainNb.ipynb

@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "d11a2343",
    "metadata": {},
    "outputs": [],
@@ -27,7 +27,8 @@
     "from typing import Dict, Union\n",
     "from pathlib import Path\n",
     "import os\n",
-    "import sys"
+    "import sys\n",
+    "import re"
    ]
   },
   {
@@ -118,17 +119,14 @@
     "\n",
     "```python\n",
     "all_demo = dfs['All_demographics_and_programs']\n",
-    "all_demo.head()\n",
+    "all_demo.head(1)\n",
     "```\n",
     "\n",
     "output:\n",
     "|col 1|col 2|col 3|\n",
     "|:--:|:--:|:--:|\n",
     "|3.14|name|apple|\n",
-    "|3.14|name|apple|\n",
-    "|3.14|name|apple|\n",
-    "|3.14|name|apple|\n",
-    "|3.14|name|apple|\n",
+    "\n",
     "\n",
     "\n",
     "#### To Remove Spaces in DataFrame name\n",
@@ -715,15 +713,68 @@
     "            print(f\"[Warning] Failed race splitting: {e}\")\n",
     "        return self\n",
     "\n",
-    "    def clean_salary(self):\n",
-    "        \"\"\"Fix salary inconsistencies.\"\"\"\n",
+    "    def clean_salary(self, hours_per_year: int = 2080):\n",
+    "        \"\"\"\n",
+    "        Clean and standardize salary values in the DataFrame.\n",
+    "\n",
+    "        Steps performed:\n",
+    "        1. Remove currency symbols, commas, and shorthand (e.g., \"$50k\" → \"50000\").\n",
+    "        2. Handle ranges by converting them to the average value \n",
+    "           (e.g., \"50,000-70,000\" → 60000).\n",
+    "        3. Convert values to numeric, coercing invalid entries to NaN.\n",
+    "        4. Treat values < 200 as hourly wages and convert to annual salaries \n",
+    "           (multiplied by `hours_per_year`).\n",
+    "        5. Drop unrealistic values greater than 1,000,000 (set to NaN).\n",
+    "\n",
+    "        Parameters\n",
+    "        ----------\n",
+    "        hours_per_year : int, optional (default=2080)\n",
+    "            Number of work hours in a year for converting hourly to annual salary.\n",
+    "\n",
+    "        Returns\n",
+    "        -------\n",
+    "        self : object\n",
+    "            The current instance with the cleaned Salary column.\n",
+    "        \"\"\"\n",
     "        try:\n",
     "            if \"Salary\" in self.df.columns:\n",
-    "                self.df[\"Salary\"] = pd.to_numeric(\n",
-    "                    self.df[\"Salary\"], errors=\"coerce\")\n",
-    "                self.df[\"Salary\"] = self.df[\"Salary\"].replace(60000, 28.84)\n",
+    "                self.df[\"Salary\"] = self.df[\"Salary\"].astype(str)\n",
+    "                def parse_salary(val: str):\n",
+    "                    val = val.strip()\n",
+    "\n",
+    "                    # Handle range like \"50k-70k\" or \"50,000–70,000\"\n",
+    "                    if \"-\" in val or \"–\" in val:\n",
+    "                        parts = re.split(r\"[-–]\", val)\n",
+    "                        nums = [parse_salary(p) for p in parts if p.strip()]\n",
+    "                        nums = [n for n in nums if n is not None]\n",
+    "                        return sum(nums) / len(nums) if nums else None\n",
+    "\n",
+    "                    # Remove $, commas, spaces\n",
+    "                    val = re.sub(r\"[\\$,]\", \"\", val)\n",
+    "\n",
+    "                    # Handle shorthand k/K (e.g., 50k -> 50000)\n",
+    "                    match = re.match(r\"(\\d+(\\.\\d+)?)([kK])\", val)\n",
+    "                    if match:\n",
+    "                        return float(match.group(1)) * 1000\n",
+    "\n",
+    "                    # Convert plain number if possible\n",
+    "                    try:\n",
+    "                        return float(val)\n",
+    "                    except ValueError:\n",
+    "                        return None\n",
+    "\n",
+    "                # Apply parsing\n",
+    "                self.df[\"Salary\"] = self.df[\"Salary\"].apply(parse_salary)\n",
+    "\n",
+    "                # Convert small numbers (hourly) to annual\n",
+    "                self.df.loc[self.df[\"Salary\"] < 200, \"Salary\"] *= hours_per_year\n",
+    "\n",
+    "                # Drop unrealistic salaries\n",
+    "                self.df.loc[self.df[\"Salary\"] > 1_000_000, \"Salary\"] = None\n",
+    "\n",
     "        except Exception as e:\n",
     "            print(f\"[Warning] Failed salary cleaning: {e}\")\n",
+    "\n",
     "        return self\n",
     "\n",
     "    def finalize(self):\n",