diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..5bc2625a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +_pycache_ \ No newline at end of file diff --git a/Exp/Exp.ipynb b/Exp/Exp.ipynb deleted file mode 100644 index ae0120bf..00000000 --- a/Exp/Exp.ipynb +++ /dev/null @@ -1,231 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - }, - "orig_nbformat": 4, - "kernelspec": { - "name": "python3", - "display_name": "Python 3.9.5 64-bit" - }, - "interpreter": { - "hash": "4f1212124a69d0e192309562875881c403391e14c64cb7bbd7e67cd7a8d12ddb" - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import tweepy\n", - "# from tweepy import api\n", - "# from tweepy import streaming\n", - "# from tweepy import auth\n", - "# from tweepy.streaming import Stream\n", - "# from autenticate import get_auth\n", - "\n", - "# class TweetsListener(tweepy.StreamListener):\n", - "# def on_connect(self):\n", - "# print(\"Estoy conectado\")\n", - "\n", - "# def on_status(self, status):\n", - "# print(status.text)\n", - "\n", - "# def on_error(self, status_code):\n", - "# print(\"Error\", status_code)\n", - "\n", - "# auth = get_auth()\n", - "# api = tweepy.API(auth, wait_on_rate_limit_notify=True , wait_on_rate_limit=True)\n", - "\n", - "# stream = TweetsListener()\n", - "# streamingApi = tweepy.Stream(auth=api.auth, listener=stream)\n", - "\n", - "# streamingApi.filter(\n", - "# track=[\"pedro castillo\", \"keiko\"]\n", - "# )\n", - "# import nltk\n", - "# nltk.download('stopwords')\n", - "\n", - "#print(\"frecuente\\xa0hsy\")" - ] - }, - { - "cell_type": "code", - "execution_count": 152, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " going to today i am it is rain\n", - "1 1 1 1 0 0 1 1 1\n", - "2 1 0 1 1 1 0 0 0\n", - "3 1 1 0 1 1 0 0 0" - ], - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
goingtotodayiamitisrain
111100111
210111000
311011000
\n
" - }, - "metadata": {}, - "execution_count": 152 - } - ], - "source": [ - "# creacion del primer data frame\n", - "dic1 = {'going': [1, 1, 1],\n", - " 'to': [1, 0, 1],\n", - " 'today': [1, 1, 0],\n", - " 'i': [0, 1, 1],\n", - " 'am': [0, 1, 1],\n", - " 'it': [1, 0, 0],\n", - " 'is': [1, 0, 0],\n", - " 'rain': [1, 0, 0]}\n", - "docs = [1,2,3]\n", - "df1 = pd.DataFrame(dic1, index=docs)\n", - "df1" - ] - }, - { - "cell_type": "code", - "execution_count": 183, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n" - ] - }, - { - "cell_type": "code", - "execution_count": 184, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - " count\ngoing 3\nto 2\ntoday 2\ni 2\nam 2\nit 1\nis 1\nrain 1\n" - ] - } - ], - "source": [ - "#print(df2.shape)\n", - "df3= count_col(df1)\n", - "print(df3)" - ] - }, - { - "cell_type": "code", - "execution_count": 163, - "metadata": {}, - "outputs": [], - "source": [ - "def WordsXDocuments(dataframe):\n", - " WD = pd.DataFrame() \n", - " col_list= list(dataframe)\n", - " dataframe[col_list].sum(axis=1)\n", - " WD['Cantidad'] = dataframe[col_list].sum(axis=1)\n", - " print(WD)\n", - "\n", - " df_tf = dataframe.copy()\n", - " for col in df_tf:\n", - " nrow = len(df_tf[col])\n", - " for i in range(0,nrow):\n", - " print(WD.loc[i+1,'Cantidad'])\n", - " df_tf.loc[i+1,col] = df_tf.loc[i+1,col] / WD.loc[i+1,'Cantidad']\n", - " return df_tf\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 164, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - " going to today i am it is rain\n1 0.166667 0.166667 0.166667 0.00 0.00 0.166667 0.166667 0.166667\n2 0.250000 0.000000 0.250000 0.25 0.25 0.000000 0.000000 0.000000\n3 0.250000 0.250000 0.000000 0.25 0.25 0.000000 0.000000 0.000000\n" - ] - } - ], - "source": [ - "WD = WordsXDocuments(df1)\n", - "print(WD)" - ] - }, - { - "cell_type": "code", - "execution_count": 173, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " idf_values\n", - "going 0.00\n", - "to 0.41\n", - "today 0.41\n", - "i 0.41\n", - "am 0.41\n", - "it 1.09\n", - "is 1.09\n", - "rain 1.09" - ], - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
idf_values
going0.00
to0.41
today0.41
i0.41
am0.41
it1.09
is1.09
rain1.09
\n
" - }, - "metadata": {}, - "execution_count": 173 - } - ], - "source": [ - "dic2 = {'idf_values': [0, 0.41, 0.41, 0.41, 0.41, 1.09, 1.09, 1.09]}\n", - "words = ['going', 'to', 'today', 'i', 'am', 'it', 'is','rain']\n", - "df2 = pd.DataFrame(dic2, index=words)\n", - "df2 " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def tf_idf(df_tf, df_idf):\n", - " df_tf_idf = df_tf.copy()\n", - " for col in df_tf_idf:\n", - " nrow = len(df_tf_idf[col])\n", - " for i in range(0,nrow):\n", - " df_tf_idf.loc[i+1,col] *= df_idf.loc[col,'idf_values']\n", - " return df_tf_idf\n" - ] - }, - { - "cell_type": "code", - "execution_count": 167, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ] -} \ No newline at end of file diff --git a/Exp/Exp.py b/Exp/Exp.py deleted file mode 100644 index 0e30065c..00000000 --- a/Exp/Exp.py +++ /dev/null @@ -1,79 +0,0 @@ -import numpy as np -import pandas as pd -import re - -#Visualización -import matplotlib.pyplot as plt -import matplotlib -from wordcloud import WordCloud, STOPWORDS - -#nltk librería de análisis de lenguaje -import nltk -#Este proceso puede hacerse antes de forma manual, descargar las stopwords de la librería nltk -nltk.download('stopwords') -from nltk.corpus import stopwords -stop_words_sp = set(stopwords.words('spanish')) -stop_words_en = set(stopwords.words('english')) -#Concatenar las stopwords aplicándose a una cuenta que genera contenido en inglés y español -stop_words = stop_words_sp | stop_words_en -from nltk import tokenize - -matplotlib.style.use('ggplot') -pd.options.mode.chained_assignment = None - -#Últimos 400 tweets previamente descargados -tweets = pd.read_csv('sample_tweets-400.csv') -#Últimos 3240 tweets previamente descargados -tweets2 = pd.read_csv('sample_tweets.csv') - -def wordcloud(tweets,col,idgraf): - #Crear la imagen con las palabras más frecuentes - wordcloud = WordCloud(background_color="white",stopwords=stop_words,random_state = 2016).generate(" ".join([i for i in tweets[col]])) - #Preparar la figura - plt.figure(num=idgraf, figsize=(20,10), facecolor='k') - plt.imshow(wordcloud) - plt.axis("off") - plt.title("Good Morning Datascience+") - - -def tweetprocess(tweets,idgraf): - #Monitorear que ha ingresado a procesar el gráfico - print(idgraf) - #Imprimir un tweet que sepamos contenga RT @, handles y puntuación para ver su eliminación - print(tweets['text'][3]) - tweets['tweetos'] = '' - - #add tweetos first part - for i in range(len(tweets['text'])): - try: - tweets['tweetos'][i] = tweets['text'].str.split(' ')[i][0] - except AttributeError: - tweets['tweetos'][i] = 'other' - - #Prepocesar tweets con 'RT @' - for i in range(len(tweets['text'])): - if tweets['tweetos'].str.contains('@')[i] == False: - tweets['tweetos'][i] = 'other' - - # Remover URLs, RTs, y twitter handles - for i in range(len(tweets['text'])): - tweets['text'][i] = " ".join([word for word in tweets['text'][i].split() - if 'http' not in word and '@' not in word and '<' not in word and 'RT' not in word]) - #Monitorear que se removieron las menciones y URLs - print("------Después de remover menciones y URLs --------") - print(tweets['text'][3]) - - #Remover puntuación, se agregan símbolos del español - tweets['text'] = tweets['text'].apply(lambda x: re.sub('[¡!@#$:).;,¿?&]', '', x.lower())) - tweets['text'] = tweets['text'].apply(lambda x: re.sub(' ', ' ', x)) - #Monitorear que se removió la puntuación y queda en minúsculas - print("------Después de remover signos de puntuación y pasar a minúsculas--------") - print(tweets['text'][3]) - #hacer el análisis de WordCloud - wordcloud(tweets,'text',idgraf) - -#Graficar tendencia 400 tweets -tweetprocess(tweets,100) -#Graficar tendencia 3240 tweets -tweetprocess(tweets2,200) -plt.show() \ No newline at end of file diff --git a/Funciones/__pycache__/AnalisisDatos.cpython-39.pyc b/Funciones/__pycache__/AnalisisDatos.cpython-39.pyc deleted file mode 100644 index 9cc2c318..00000000 Binary files a/Funciones/__pycache__/AnalisisDatos.cpython-39.pyc and /dev/null differ diff --git a/Funciones/__pycache__/LimpiezaDatos.cpython-39.pyc b/Funciones/__pycache__/LimpiezaDatos.cpython-39.pyc deleted file mode 100644 index 88b4f098..00000000 Binary files a/Funciones/__pycache__/LimpiezaDatos.cpython-39.pyc and /dev/null differ diff --git a/Funciones/__pycache__/ReduccionComplejidad.cpython-39.pyc b/Funciones/__pycache__/ReduccionComplejidad.cpython-39.pyc deleted file mode 100644 index 44d20282..00000000 Binary files a/Funciones/__pycache__/ReduccionComplejidad.cpython-39.pyc and /dev/null differ diff --git a/__pycache__/ExtraerTweets.cpython-39.pyc b/__pycache__/ExtraerTweets.cpython-39.pyc deleted file mode 100644 index ad2a0f84..00000000 Binary files a/__pycache__/ExtraerTweets.cpython-39.pyc and /dev/null differ diff --git a/__pycache__/Ntweets.cpython-39.pyc b/__pycache__/Ntweets.cpython-39.pyc deleted file mode 100644 index 484c2ea0..00000000 Binary files a/__pycache__/Ntweets.cpython-39.pyc and /dev/null differ diff --git a/__pycache__/ProcesarTema.cpython-39.pyc b/__pycache__/ProcesarTema.cpython-39.pyc deleted file mode 100644 index d4a3ea3e..00000000 Binary files a/__pycache__/ProcesarTema.cpython-39.pyc and /dev/null differ diff --git a/__pycache__/Stanza.cpython-39.pyc b/__pycache__/Stanza.cpython-39.pyc deleted file mode 100644 index 7b5e364d..00000000 Binary files a/__pycache__/Stanza.cpython-39.pyc and /dev/null differ diff --git a/__pycache__/autenticate.cpython-39.pyc b/__pycache__/autenticate.cpython-39.pyc deleted file mode 100644 index 542ee2b1..00000000 Binary files a/__pycache__/autenticate.cpython-39.pyc and /dev/null differ