Skip to content

Commit 618f295

Browse files
NicholasJCampbellNicholasJCampbell
committed
data cleaning, near the end but need to make the function
1 parent 5fff4bc commit 618f295

1 file changed

Lines changed: 116 additions & 148 deletions

File tree

‎data/demographicscleaning.ipynb‎

Lines changed: 116 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
},
1515
{
1616
"cell_type": "code",
17-
"execution_count": 49,
17+
"execution_count": 1,
1818
"id": "9b3e31c5",
1919
"metadata": {},
2020
"outputs": [],
@@ -26,7 +26,7 @@
2626
},
2727
{
2828
"cell_type": "code",
29-
"execution_count": 67,
29+
"execution_count": 9,
3030
"id": "a7d06e21",
3131
"metadata": {},
3232
"outputs": [],
@@ -39,6 +39,49 @@
3939
"execution_count": null,
4040
"id": "275473f5",
4141
"metadata": {},
42+
"outputs": [
43+
{
44+
"ename": "NameError",
45+
"evalue": "name 'df_clean' is not defined",
46+
"output_type": "error",
47+
"traceback": [
48+
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
49+
"\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
50+
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 21\u001b[39m\n\u001b[32m 18\u001b[39m df_clean.drop_duplicates()\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df_clean\n\u001b[32m---> \u001b[39m\u001b[32m21\u001b[39m \u001b[43mdf_clean\u001b[49m.head()\n",
51+
"\u001b[31mNameError\u001b[39m: name 'df_clean' is not defined"
52+
]
53+
}
54+
],
55+
"source": [
56+
"# drop the first name and last name columns since they don't have any names\n",
57+
"# drop single parent because it is empty (only 1 row has 'No')\n",
58+
"# drop Ethnicity Hispanic/Latino because it is all null\n",
59+
"# drop Ex-Offender because only 1 row has No and the rest are null\n",
60+
"# combined Transgender M to F and F to M to the category 'Transgender' because there were only \n",
61+
"# 16 that identifed M to F or F to M\n",
62+
"\n",
63+
"# Still trying to figure out how to put it all in a function!\n",
64+
"\n",
65+
"# def data_cleaning(df_clean):\n",
66+
"# df_clean = df.drop(columns=['First Name','Last Name','Ethnicity Hispanic/Latino',\n",
67+
"# 'Single Parent','Ex-Offender','Program: Program Name'])\n",
68+
"# df_clean = df_clean['Gender'].str.replace('Transgender female to male','Transgender')\n",
69+
"# df_clean = df_clean['Gender'].str.replace('Transgender male to female','Transgender')\n",
70+
"# splitting = df_clean['Race'].str.split(';', expand=True)\n",
71+
"# splitting.columns = [f'Race_{i+1}' for i in range(splitting.shape[1])]\n",
72+
"# df_clean = pd.concat([df_clean, splitting], axis=1)\n",
73+
"# df_clean.drop(columns=['Race','Outcome'])\n",
74+
"# df_clean.drop_duplicates()\n",
75+
"# return df_clean\n",
76+
"\n",
77+
"# df_clean.head()"
78+
]
79+
},
80+
{
81+
"cell_type": "code",
82+
"execution_count": null,
83+
"id": "6b09534b",
84+
"metadata": {},
4285
"outputs": [
4386
{
4487
"data": {
@@ -63,7 +106,6 @@
63106
" <th></th>\n",
64107
" <th>Auto Id</th>\n",
65108
" <th>Gender</th>\n",
66-
" <th>Outcome</th>\n",
67109
" <th>Veteran</th>\n",
68110
" <th>Justice Involved</th>\n",
69111
" <th>Race_1</th>\n",
@@ -77,31 +119,6 @@
77119
" <th>0</th>\n",
78120
" <td>202107-1206</td>\n",
79121
" <td>Male</td>\n",
80-
" <td>NaN</td>\n",
81-
" <td>No</td>\n",
82-
" <td>NaN</td>\n",
83-
" <td>Black or African American</td>\n",
84-
" <td>None</td>\n",
85-
" <td>None</td>\n",
86-
" <td>None</td>\n",
87-
" </tr>\n",
88-
" <tr>\n",
89-
" <th>1</th>\n",
90-
" <td>202107-1206</td>\n",
91-
" <td>Male</td>\n",
92-
" <td>NaN</td>\n",
93-
" <td>No</td>\n",
94-
" <td>NaN</td>\n",
95-
" <td>Black or African American</td>\n",
96-
" <td>None</td>\n",
97-
" <td>None</td>\n",
98-
" <td>None</td>\n",
99-
" </tr>\n",
100-
" <tr>\n",
101-
" <th>2</th>\n",
102-
" <td>202107-1206</td>\n",
103-
" <td>Male</td>\n",
104-
" <td>NaN</td>\n",
105122
" <td>No</td>\n",
106123
" <td>NaN</td>\n",
107124
" <td>Black or African American</td>\n",
@@ -113,7 +130,6 @@
113130
" <th>3</th>\n",
114131
" <td>202108-5167</td>\n",
115132
" <td>Male</td>\n",
116-
" <td>Successfully Completed</td>\n",
117133
" <td>No</td>\n",
118134
" <td>No</td>\n",
119135
" <td>Asian</td>\n",
@@ -127,180 +143,132 @@
127143
" <td>Male</td>\n",
128144
" <td>NaN</td>\n",
129145
" <td>NaN</td>\n",
130-
" <td>NaN</td>\n",
131146
" <td>Black or African American</td>\n",
132147
" <td>None</td>\n",
133148
" <td>None</td>\n",
134149
" <td>None</td>\n",
135150
" </tr>\n",
136151
" <tr>\n",
137-
" <th>...</th>\n",
138-
" <td>...</td>\n",
139-
" <td>...</td>\n",
140-
" <td>...</td>\n",
141-
" <td>...</td>\n",
142-
" <td>...</td>\n",
143-
" <td>...</td>\n",
144-
" <td>...</td>\n",
145-
" <td>...</td>\n",
146-
" <td>...</td>\n",
147-
" </tr>\n",
148-
" <tr>\n",
149-
" <th>32225</th>\n",
150-
" <td>202502-20671</td>\n",
152+
" <th>5</th>\n",
153+
" <td>202108-5172</td>\n",
151154
" <td>Female</td>\n",
152155
" <td>NaN</td>\n",
153156
" <td>NaN</td>\n",
154-
" <td>NaN</td>\n",
155157
" <td>White</td>\n",
156158
" <td>None</td>\n",
157159
" <td>None</td>\n",
158160
" <td>None</td>\n",
159161
" </tr>\n",
160162
" <tr>\n",
161-
" <th>32226</th>\n",
162-
" <td>202410-17602</td>\n",
163+
" <th>6</th>\n",
164+
" <td>202107-1208</td>\n",
163165
" <td>Female</td>\n",
164166
" <td>NaN</td>\n",
165167
" <td>NaN</td>\n",
166-
" <td>NaN</td>\n",
167-
" <td>White</td>\n",
168-
" <td>None</td>\n",
169-
" <td>None</td>\n",
170-
" <td>None</td>\n",
171-
" </tr>\n",
172-
" <tr>\n",
173-
" <th>32227</th>\n",
174-
" <td>202506-23809</td>\n",
175-
" <td>Female</td>\n",
176-
" <td>NaN</td>\n",
177-
" <td>NaN</td>\n",
178-
" <td>NaN</td>\n",
179-
" <td>White</td>\n",
180-
" <td>None</td>\n",
181-
" <td>None</td>\n",
182-
" <td>None</td>\n",
183-
" </tr>\n",
184-
" <tr>\n",
185-
" <th>32228</th>\n",
186-
" <td>202410-17749</td>\n",
187-
" <td>Female</td>\n",
188-
" <td>NaN</td>\n",
189-
" <td>NaN</td>\n",
190-
" <td>NaN</td>\n",
191-
" <td>White</td>\n",
192-
" <td>None</td>\n",
193-
" <td>None</td>\n",
194-
" <td>None</td>\n",
195-
" </tr>\n",
196-
" <tr>\n",
197-
" <th>32229</th>\n",
198-
" <td>202505-23270</td>\n",
199-
" <td>Male</td>\n",
200-
" <td>NaN</td>\n",
201-
" <td>NaN</td>\n",
202-
" <td>NaN</td>\n",
203-
" <td>White</td>\n",
168+
" <td>Black or African American</td>\n",
204169
" <td>None</td>\n",
205170
" <td>None</td>\n",
206171
" <td>None</td>\n",
207172
" </tr>\n",
208173
" </tbody>\n",
209174
"</table>\n",
210-
"<p>32230 rows × 9 columns</p>\n",
211175
"</div>"
212176
],
213177
"text/plain": [
214-
" Auto Id Gender Outcome Veteran Justice Involved \\\n",
215-
"0 202107-1206 Male NaN No NaN \n",
216-
"1 202107-1206 Male NaN No NaN \n",
217-
"2 202107-1206 Male NaN No NaN \n",
218-
"3 202108-5167 Male Successfully Completed No No \n",
219-
"4 202108-5171 Male NaN NaN NaN \n",
220-
"... ... ... ... ... ... \n",
221-
"32225 202502-20671 Female NaN NaN NaN \n",
222-
"32226 202410-17602 Female NaN NaN NaN \n",
223-
"32227 202506-23809 Female NaN NaN NaN \n",
224-
"32228 202410-17749 Female NaN NaN NaN \n",
225-
"32229 202505-23270 Male NaN NaN NaN \n",
178+
" Auto Id Gender Veteran Justice Involved Race_1 \\\n",
179+
"0 202107-1206 Male No NaN Black or African American \n",
180+
"3 202108-5167 Male No No Asian \n",
181+
"4 202108-5171 Male NaN NaN Black or African American \n",
182+
"5 202108-5172 Female NaN NaN White \n",
183+
"6 202107-1208 Female NaN NaN Black or African American \n",
226184
"\n",
227-
" Race_1 Race_2 Race_3 Race_4 \n",
228-
"0 Black or African American None None None \n",
229-
"1 Black or African American None None None \n",
230-
"2 Black or African American None None None \n",
231-
"3 Asian None None None \n",
232-
"4 Black or African American None None None \n",
233-
"... ... ... ... ... \n",
234-
"32225 White None None None \n",
235-
"32226 White None None None \n",
236-
"32227 White None None None \n",
237-
"32228 White None None None \n",
238-
"32229 White None None None \n",
239-
"\n",
240-
"[32230 rows x 9 columns]"
185+
" Race_2 Race_3 Race_4 \n",
186+
"0 None None None \n",
187+
"3 None None None \n",
188+
"4 None None None \n",
189+
"5 None None None \n",
190+
"6 None None None "
241191
]
242192
},
243-
"execution_count": 69,
193+
"execution_count": 18,
244194
"metadata": {},
245195
"output_type": "execute_result"
246196
}
247197
],
248198
"source": [
249-
"# drop the first name and last name columns since they don't have any names\n",
250-
"# drop single parent because it is empty (only 1 row has 'No')\n",
251-
"# drop Ethnicity Hispanic/Latino because it is all null\n",
252-
"# drop Ex-Offender because only 1 row has No and the rest are null\n",
253-
"# combined Transgender M to F and F to M to the category 'Transgender' because there were only \n",
254-
"# 16 that identifed M to F or F to M\n",
255-
"\n",
199+
"df_clean = df.drop(columns=['First Name','Last Name','Ethnicity Hispanic/Latino',\n",
200+
" 'Single Parent','Ex-Offender','Program: Program Name','Outcome'])\n",
201+
"df_clean['Gender'].str.replace('Transgender female to male','Transgender')\n",
202+
"df_clean['Gender'].str.replace('Transgender male to female','Transgender')\n",
203+
"splitting = df_clean['Race'].str.split(';', expand=True)\n",
204+
"splitting.columns = [f'Race_{i+1}' for i in range(splitting.shape[1])]\n",
205+
"df_clean = pd.concat([df_clean, splitting], axis=1)\n",
206+
"df_clean = df_clean.drop(columns=['Race'])\n",
207+
"#removing full duplicates which successfully removes all the auto id duplicates as well\n",
208+
"df_cleanest = df_clean.drop_duplicates()\n",
256209
"\n",
257-
"def data_cleaning(df_clean):\n",
258-
" df_clean = df.drop(columns=['First Name','Last Name','Ethnicity Hispanic/Latino',\n",
259-
" 'Single Parent','Ex-Offender','Program: Program Name'])\n",
260-
" df_clean = df_clean['Gender'].str.replace('Transgender female to male','Transgender')\n",
261-
" df_clean = df_clean['Gender'].str.replace('Transgender male to female','Transgender')\n",
262-
" splitting = df_clean['Race'].str.split(';', expand=True)\n",
263-
" splitting.columns = [f'Race_{i+1}' for i in range(splitting.shape[1])]\n",
264-
" df_clean = pd.concat([df_clean, splitting], axis=1)\n",
265-
" df_clean.drop(columns=['Race'])\n",
266-
" return(df_clean)\n",
267-
"\n",
268-
"df_clean"
210+
"df_cleanest.head()"
269211
]
270212
},
271213
{
272214
"cell_type": "code",
273-
"execution_count": 70,
215+
"execution_count": 20,
274216
"id": "95ff80e1",
275217
"metadata": {},
276218
"outputs": [
277219
{
278220
"data": {
221+
"text/html": [
222+
"<div>\n",
223+
"<style scoped>\n",
224+
" .dataframe tbody tr th:only-of-type {\n",
225+
" vertical-align: middle;\n",
226+
" }\n",
227+
"\n",
228+
" .dataframe tbody tr th {\n",
229+
" vertical-align: top;\n",
230+
" }\n",
231+
"\n",
232+
" .dataframe thead th {\n",
233+
" text-align: right;\n",
234+
" }\n",
235+
"</style>\n",
236+
"<table border=\"1\" class=\"dataframe\">\n",
237+
" <thead>\n",
238+
" <tr style=\"text-align: right;\">\n",
239+
" <th></th>\n",
240+
" <th>Auto Id</th>\n",
241+
" <th>Gender</th>\n",
242+
" <th>Veteran</th>\n",
243+
" <th>Justice Involved</th>\n",
244+
" <th>Race_1</th>\n",
245+
" <th>Race_2</th>\n",
246+
" <th>Race_3</th>\n",
247+
" <th>Race_4</th>\n",
248+
" </tr>\n",
249+
" </thead>\n",
250+
" <tbody>\n",
251+
" </tbody>\n",
252+
"</table>\n",
253+
"</div>"
254+
],
279255
"text/plain": [
280-
"Auto Id\n",
281-
"202210-9836 69\n",
282-
"202302-10868 51\n",
283-
"202206-8677 47\n",
284-
"202501-19937 46\n",
285-
"202110-5548 40\n",
286-
" ..\n",
287-
"202211-10136 1\n",
288-
"202210-9854 1\n",
289-
"202210-9659 1\n",
290-
"202210-9906 1\n",
291-
"202208-9170 1\n",
292-
"Name: count, Length: 9208, dtype: int64"
256+
"Empty DataFrame\n",
257+
"Columns: [Auto Id, Gender, Veteran, Justice Involved, Race_1, Race_2, Race_3, Race_4]\n",
258+
"Index: []"
293259
]
294260
},
295-
"execution_count": 70,
261+
"execution_count": 20,
296262
"metadata": {},
297263
"output_type": "execute_result"
298264
}
299265
],
300266
"source": [
301267
"# what does 'not applicable' mean for Veteran column\n",
302268
"# what does Justice Involved column mean?\n",
303-
"df_clean['Auto Id'].value_counts()"
269+
"df_id_duplicates = df_cleanest[df_cleanest.duplicated('Auto Id', keep=False)].sort_values('Auto Id')\n",
270+
"df_id_duplicates\n",
271+
"# df_cleanest['Auto Id'].value_counts()"
304272
]
305273
}
306274
],

0 commit comments

Comments
 (0)