Skip to content

Commit d684cf8

Browse files
dmorton714dmorton714
committed
cleaned up repo
1 parent 7661174 commit d684cf8

2 files changed

Lines changed: 15 additions & 385 deletions

File tree

‎src/notebooks/ideal.ipynb‎

Lines changed: 15 additions & 265 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@
6161
"metadata": {},
6262
"source": [
6363
"## Update cleaning code \n",
64-
"Look at our cleaning code that we have. \n",
65-
"we should start to make changes to it to account for this. \n",
66-
"We need to make it so it so the program doesn't crash when something fails \n",
64+
"- Look at our cleaning code that we have. \n",
65+
"- we should start to make changes to it to account for this. \n",
66+
"- We need to make it so it so the program doesn't crash when something fails \n",
6767
" - [Try Except logic updates](https://www.w3schools.com/python/python_try_except.asp)\n",
6868
" - make the messages mean something meaningful\n",
69-
"Ideally we will not drop anything from our data "
69+
"- Ideally we will not drop anything from our data \n"
7070
]
7171
},
7272
{
@@ -76,125 +76,12 @@
7676
"metadata": {},
7777
"outputs": [],
7878
"source": [
79-
"class DemographicsCleaning:\n",
80-
" \"\"\"\n",
81-
" A class for cleaning and preprocessing demographic data.\n",
82-
"\n",
83-
" Provides methods to:\n",
84-
" - Remove unused or mostly null columns\n",
85-
" - Normalize gender values\n",
86-
" - Split the 'Race' column into multiple race columns\n",
87-
" - Drop duplicate rows\n",
88-
" \"\"\"\n",
89-
"\n",
90-
" @staticmethod\n",
91-
" def remove_unused_columns(df: pd.DataFrame) -> pd.DataFrame:\n",
92-
" \"\"\"\n",
93-
" Remove columns with mostly null values or unnecessary information.\n",
94-
"\n",
95-
" Args:\n",
96-
" df (pd.DataFrame): Input dataframe containing demographic data.\n",
97-
"\n",
98-
" Returns:\n",
99-
" pd.DataFrame: Dataframe with specified columns removed.\n",
100-
" \"\"\"\n",
101-
" columns_to_drop = [\n",
102-
" 'First Name', 'Last Name', 'Ethnicity Hispanic/Latino',\n",
103-
" 'Single Parent', 'Ex-Offender', 'Program: Program Name', 'Outcome'\n",
104-
" ]\n",
105-
" return df.drop(columns=columns_to_drop, errors='ignore')\n",
106-
"\n",
107-
" @staticmethod\n",
108-
" def normalize_gender(df: pd.DataFrame) -> pd.DataFrame:\n",
109-
" \"\"\"\n",
110-
" Normalize gender values by combining 'Transgender male to female'\n",
111-
" and 'Transgender female to male' into a single 'Transgender' category.\n",
112-
"\n",
113-
" Args:\n",
114-
" df (pd.DataFrame): Input dataframe containing a 'Gender' column.\n",
115-
"\n",
116-
" Returns:\n",
117-
" pd.DataFrame: Dataframe with normalized gender values.\n",
118-
" \"\"\"\n",
119-
" df['Gender'] = df['Gender'].replace({\n",
120-
" 'Transgender male to female': 'Transgender',\n",
121-
" 'Transgender female to male': 'Transgender'\n",
122-
" })\n",
123-
" return df\n",
124-
"\n",
125-
" @staticmethod\n",
126-
" def split_race_column(df: pd.DataFrame) -> pd.DataFrame:\n",
127-
" \"\"\"\n",
128-
" Split the 'Race' column into multiple columns\n",
129-
" if multiple races are selected.\n",
130-
"\n",
131-
" Args:\n",
132-
" df (pd.DataFrame): Input dataframe containing a 'Race' column.\n",
133-
"\n",
134-
" Returns:\n",
135-
" pd.DataFrame: Dataframe with new columns Race_1, Race_2, etc.\n",
136-
" \"\"\"\n",
137-
" splitting = df['Race'].str.split(';', expand=True)\n",
138-
" splitting.columns = [f'Race_{i+1}' for i in range(splitting.shape[1])]\n",
139-
" df = pd.concat([df.drop(columns=['Race']), splitting], axis=1)\n",
140-
" return df\n",
141-
"\n",
142-
" @staticmethod\n",
143-
" def drop_duplicates(df: pd.DataFrame) -> pd.DataFrame:\n",
144-
" \"\"\"\n",
145-
" Remove duplicate rows from the dataframe.\n",
146-
"\n",
147-
" Args:\n",
148-
" df (pd.DataFrame): Input dataframe.\n",
149-
"\n",
150-
" Returns:\n",
151-
" pd.DataFrame: Dataframe without duplicate rows.\n",
152-
" \"\"\"\n",
153-
" return df.drop_duplicates()\n",
154-
"\n",
155-
" @classmethod\n",
156-
" def clean(cls, df: pd.DataFrame) -> pd.DataFrame:\n",
157-
" \"\"\"\n",
158-
" Perform the full data cleaning process on demographics data.\n",
159-
"\n",
160-
" Steps include:\n",
161-
" - Removing unused or mostly null columns\n",
162-
" - Normalizing gender values\n",
163-
" - Splitting the 'Race' column into multiple race columns\n",
164-
" - Dropping duplicate rows\n",
165-
"\n",
166-
" Args:\n",
167-
" df (pd.DataFrame): Raw demographics dataframe.\n",
168-
"\n",
169-
" Returns:\n",
170-
" pd.DataFrame: Cleaned dataframe ready for analysis.\n",
171-
" \"\"\"\n",
172-
" df = cls.remove_unused_columns(df)\n",
173-
" df = cls.normalize_gender(df)\n",
174-
" df = cls.split_race_column(df)\n",
175-
" df = cls.drop_duplicates(df)\n",
176-
" return df\n",
177-
"\n",
178-
"\n",
179-
"class WorceCleaning:\n",
180-
" \"\"\"\n",
181-
" A placeholder for a class that can be used to clean Worce data.\n",
182-
" This class can be extended in the future to include specific cleaning methods.\n",
183-
" \"\"\"\n",
184-
" @staticmethod\n",
185-
" def clean(df: pd.DataFrame) -> pd.DataFrame:\n",
186-
" \"\"\"\n",
187-
" Placeholder method for cleaning Worce data.\n",
188-
" Currently does nothing but can be extended in the future.\n",
189-
"\n",
190-
" Args:\n",
191-
" df (pd.DataFrame): Input dataframe containing Worce data.\n",
192-
"\n",
193-
" Returns:\n",
194-
" pd.DataFrame: Unchanged dataframe.\n",
195-
" \"\"\"\n",
196-
" pass\n",
197-
"\n"
79+
"'''\n",
80+
"See the functions in files:\n",
81+
"- src/Carmen_WORCEmployment.py\n",
82+
"- src/cleaning_enrollments_data.py\n",
83+
"- src/cleaning.py\n",
84+
"'''"
19885
]
19986
},
20087
{
@@ -230,7 +117,11 @@
230117
"- Look at the various plots \n",
231118
"- make a consistent color scheme\n",
232119
"- pick the plots that go with the report above \n",
233-
"- make missing plots \n"
120+
"- make missing plots \n",
121+
"- make plots have the option to show & save in the functions\n",
122+
"\n",
123+
"see `src/notebooks/visualization_examples.ipynb`\n",
124+
"See below from `src/Carmen_WORCEmployment_Plots.py`"
234125
]
235126
},
236127
{
@@ -276,147 +167,6 @@
276167
" plt.show()"
277168
]
278169
},
279-
{
280-
"cell_type": "code",
281-
"execution_count": 1,
282-
"id": "8f471c68",
283-
"metadata": {},
284-
"outputs": [
285-
{
286-
"ename": "ModuleNotFoundError",
287-
"evalue": "No module named 'most_common_pathways_taken_data'",
288-
"output_type": "error",
289-
"traceback": [
290-
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
291-
"\u001b[31mModuleNotFoundError\u001b[39m Traceback (most recent call last)",
292-
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 17\u001b[39m\n\u001b[32m 15\u001b[39m sys.path.append(parent_dir)\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mdash\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Dash, dcc, html, Input, Output\n\u001b[32m---> \u001b[39m\u001b[32m17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mmost_common_pathways_taken_data\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Most_common_pathways_taken_data\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mcompletion_rate_data\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Completion_rate_data\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mcleaning_enrollments_data\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m EnrollmentsCleaning\n",
293-
"\u001b[31mModuleNotFoundError\u001b[39m: No module named 'most_common_pathways_taken_data'"
294-
]
295-
}
296-
],
297-
"source": [
298-
"# %% [markdown]\n",
299-
"# # Visualization examples\n",
300-
"# \n",
301-
"# Visualizion was not turn into a class because the project will use Google Locker for dashboard creation, this notebook only works to showcase how to use the Data Manipulation classes.\n",
302-
"\n",
303-
"# %% [markdown]\n",
304-
"# ## Imports\n",
305-
"\n",
306-
"# %%\n",
307-
"import pandas as pd\n",
308-
"import plotly.express as px\n",
309-
"import os\n",
310-
"import sys\n",
311-
"parent_dir = os.path.abspath(\"..\")\n",
312-
"sys.path.append(parent_dir)\n",
313-
"from dash import Dash, dcc, html, Input, Output\n",
314-
"from most_common_pathways_taken_data import Most_common_pathways_taken_data\n",
315-
"from completion_rate_data import Completion_rate_data\n",
316-
"from cleaning_enrollments_data import EnrollmentsCleaning\n",
317-
"\n",
318-
"# %% [markdown]\n",
319-
"# ## Cleaning data\n",
320-
"# \n",
321-
"# This step should be done before the use of any of the Data classes\n",
322-
"\n",
323-
"# %%\n",
324-
"cleaner = EnrollmentsCleaning(pd.read_excel('../../data/ARC_Enrollments.xlsx'))\n",
325-
"\n",
326-
"\n",
327-
"# %% [markdown]\n",
328-
"# ## Most common pathway taken:\n",
329-
"\n",
330-
"# %%\n",
331-
"def Dash_most_selected_path_by_cohort() -> Dash: # Need to pass the dataframe argument because of how the Data is structure\n",
332-
" app = Dash(__name__)\n",
333-
" # Const\n",
334-
" data_class = Most_common_pathways_taken_data(cleaner.Get_clean_data())\n",
335-
"\n",
336-
" dropdown_options = data_class.Get_cohorts_list()\n",
337-
" pathway_color = {\n",
338-
" 'Web Development M1': 'blue',\n",
339-
" 'Data Analysis M1': 'red', \n",
340-
" 'Software Development M1': 'green',\n",
341-
" 'Quality Assurance M1': 'yellow', \n",
342-
" 'User Experience M1': 'purple'\n",
343-
" }\n",
344-
"\n",
345-
" # Display\n",
346-
" app.layout = html.Div([\n",
347-
" html.H2('Cohorts', style={'text-align': \"center\"}),\n",
348-
" html.P('Select Cohort:'),\n",
349-
" dcc.Dropdown(\n",
350-
" id=\"dropdown\",\n",
351-
" options=dropdown_options,\n",
352-
" value=dropdown_options[0],\n",
353-
" clearable=False,\n",
354-
" ),\n",
355-
" dcc.Graph(id=\"graph\")\n",
356-
" \n",
357-
" ], style={'backgroundColor':'white'})\n",
358-
"\n",
359-
" @app.callback(\n",
360-
" Output(\"graph\", \"figure\"),\n",
361-
" Input(\"dropdown\", \"value\"))\n",
362-
"\n",
363-
" # Graph\n",
364-
" def tt(time):\n",
365-
" df = data_class.Get_data_by_cohort(time)\n",
366-
" fig = px.pie(df, names='Service', values='count', color='Service', color_discrete_map=pathway_color)\n",
367-
" return fig\n",
368-
"\n",
369-
" return app\n",
370-
"\n",
371-
" # TODO: Add number of students per each cohort \n",
372-
" # TODO: Fix the options on the selection \n",
373-
" # TODO: make colors better\n",
374-
"\n",
375-
"Dash_most_selected_path_by_cohort().run(debug=True, port=8052)\n",
376-
"\n",
377-
"# %% [markdown]\n",
378-
"# ## Compleation rates:\n",
379-
"\n",
380-
"# %%\n",
381-
"def Dash_completion_rates_by_path() -> Dash: # TODO: fix data structure so visualization doesn't use df\n",
382-
" app2 = Dash(__name__)\n",
383-
" # Const\n",
384-
" data_class = Completion_rate_data(cleaner.Get_clean_data())\n",
385-
" completion_df = data_class.Get_completion_percentages().round(2)\n",
386-
" options = data_class.Get_pathways_name(completion_df)\n",
387-
"\n",
388-
" # Display\n",
389-
" app2.layout = html.Div([\n",
390-
" html.H2('Pathways Completion', style={'text-align': \"center\"}),\n",
391-
" html.P('Select pathway:'),\n",
392-
" dcc.Dropdown(\n",
393-
" id=\"dropdown\",\n",
394-
" options=options,\n",
395-
" value=options[0],\n",
396-
" clearable=False,\n",
397-
" ),\n",
398-
" dcc.Graph(id=\"graph\")\n",
399-
" \n",
400-
" ], style={'backgroundColor':'white'})\n",
401-
"\n",
402-
" @app2.callback(\n",
403-
" Output(\"graph\", \"figure\"),\n",
404-
" Input(\"dropdown\", \"value\"))\n",
405-
"\n",
406-
" # Graph\n",
407-
" # TODO: Need to add an extra selection box with the cohorts\n",
408-
" def Display_pathway_completion(p):\n",
409-
" df = completion_df[completion_df['Pathway'] == p]\n",
410-
" fig = px.bar(df, x='Module', y='Successfully Completed')\n",
411-
" return fig\n",
412-
"\n",
413-
" return app2\n",
414-
"\n",
415-
"Dash_completion_rates_by_path().run(debug=True, port=8053)\n",
416-
"\n",
417-
"\n"
418-
]
419-
},
420170
{
421171
"cell_type": "markdown",
422172
"id": "f905708f",

0 commit comments

Comments
 (0)