Twitter - Financial News Scraper, VADER Sentiment Analysis

from tweepy import Stream
from tweepy import OAuthHandler
from tweepy import StreamListener
import tweepy
import json
import ascii

ckey = "ckey"
csecret = "csecret"
atoken = "atoken"
asecret = "asecret"

analyzer = SentimentIntensityAnalyzer() # Use the VADER Sentiment Analyzer

class mslistener(StreamListener):

    def on_data(self, data):
        try:
            all_data = json.loads(data)
            tweet = all_data["text"]
             
            sentiment = analyzer.polarity_scores(tweet)['compound']
            if sentiment > 0:
                sentiment_value = "pos"
            else:
                sentiment_value = "neg"
            print(tweet, sentiment_value)

            output = open("twitter-currout.txt", "a")
            output.write(sentiment_value)
            output.write('\n')
            output.close()
        except:
            pass
        return(True)

        
    def on_error(self, status):
        print("Error:", status)
        
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)

twitterStream = Stream(auth, listener = mslistener())
twitterStream.filter(track=["trump"])

import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import time
style.use("ggplot")

fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)


def animate(i):
    pullData1 = open("twitter-currout.txt","r").read()
    
    lines1 = pullData1.split('\n')
   
    xar1 = []
    yar1 = []
    
    x1 = 0
    y1 = 0
    
    for l in lines1:
        x1 += 1
        if "pos" in l:
            y1 += 1
        elif "neg" in l:
            y1 -= 1
            
        xar1.append(x1)
        yar1.append(y1)

    ax1.clear()
    ax1.plot(xar1, yar1)
    ax1.set_ylabel("Sentiment")
    ax1.set_xlabel("Number of Tweets")

ani = animation.FuncAnimation(fig, animate, interval = 1000)

plt.show()

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def twitter_cleaner(file_path):
    df = pd.read_csv(file_path, delimiter=";")
    df = pd.DataFrame(df.ix[:,0].str.split(";").tolist())
    df = df.iloc[:, 0:10]
    df.columns = ["Username", "Date", "Retweets", "Favourites", "Tweet", "Geo", "Mentions", "Hastags", "ID", "Permalink"]
    df.Date = pd.to_datetime(df.Date)
    df['Short Date'] = df['Date'].apply(lambda x: x.date().strftime("%Y-%m-%d")) 
    df.Tweet = df.Tweet.apply(lambda x: re.sub('http(.*)', "", x))

    return(df)

wsj_df = twitter_cleaner("C:/Users/Clint_PC/Downloads/GetOldTweets-python-master/output_wsj_clean.csv")
afr_df = twitter_cleaner("C:/Users/Clint_PC/Downloads/GetOldTweets-python-master/output_afr_clean.csv")

fig = plt.figure(figsize = (20,15))
plt.suptitle("Tweet Counts", size = 25, fontweight='bold')

plt.subplot(321)
ax=wsj_df.groupby(wsj_df.Date.map(lambda t: t.minute)).Tweet.count().plot()
afr_df.groupby(afr_df.Date.map(lambda t: t.minute)).Tweet.count().plot(ax=ax)
plt.xlabel("Minute")
plt.ylabel("Tweets")
ax.legend(["WSJ", "AFR"])

plt.subplot(322)
ax=wsj_df.groupby(wsj_df.Date.map(lambda t: t.hour)).Tweet.count().plot()
afr_df.groupby(afr_df.Date.map(lambda t: t.hour)).Tweet.count().plot(ax=ax)
plt.xlabel("Hour")

plt.subplot(323)
ax=wsj_df.groupby(wsj_df.Date.map(lambda t: t.day)).Tweet.count().plot()
afr_df.groupby(afr_df.Date.map(lambda t: t.day)).Tweet.count().plot(ax=ax)
plt.xlabel("Day")

plt.subplot(324)
ax=wsj_df.groupby(wsj_df.Date.map(lambda t: t.month)).Tweet.count().plot()
afr_df.groupby(afr_df.Date.map(lambda t: t.month)).Tweet.count().plot(ax=ax)
plt.xlabel("Month")

plt.subplot(325)
ax=wsj_df.groupby(wsj_df.Date.map(lambda t: t.year)).Tweet.count().plot()
afr_df.groupby(afr_df.Date.map(lambda t: t.year)).Tweet.count().plot(ax=ax)
plt.xlabel("Year")

plt.tight_layout(rect=[0, 0.1, 1, 0.97])
plt.show()

from nltk.sentiment.vader import SentimentIntensityAnalyzer # VADER https://github.com/cjhutto/vaderSentiment
from nltk import tokenize

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from nltk.corpus import stopwords
stop = set(stopwords.words('english'))
# Create a single string for each date (since we only want to look at word counts)
combined_tweets = afr_df.groupby(['Short Date']).Tweet.apply(lambda x: ' '.join(x).lower().strip()).reset_index()

combined_tweets.Tweet = combined_tweets.Tweet.str.replace("[", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("]", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("\\", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("@", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("\\,", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("\t", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("pic", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("'", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("sub", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("reqd", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("’", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("\"", "")
combined_tweets.Tweet = combined_tweets.Tweet.str.replace("…", "")

combined_tweets_string= str(tuple(combined_tweets.Tweet.tolist()))
#vectorizer = CountVectorizer()
#news_vect = vectorizer.build_tokenizer()(combined_tweets_string)
#word_counts = pd.DataFrame([[x,news_vect.count(x)] for x in set(news_vect)], columns = ['Word', 'Count'])

from wordcloud import WordCloud

# lower max_font_size
wordcloud = WordCloud(max_font_size=40).generate(combined_tweets_string)
plt.figure(figsize = (30,10))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()

scores = pd.DataFrame(index = combined_tweets['Short Date'], columns = ['Compound', 'Positive', 'Negative', "Neutral"])

analyzer = SentimentIntensityAnalyzer() # Use the VADER Sentiment Analyzer

for j in range(1,combined_tweets.shape[0]):    
    tmp_neu = 0
    tmp_neg = 0
    tmp_pos = 0
    tmp_comp = 0
    text = afr_df.iloc[j,1]
    if(str(text) == "nan"):
        tmp_comp +=  0
        tmp_neg += 0
        tmp_neu += 0
        tmp_pos += 0
    else:
        vs = analyzer.polarity_scores(combined_tweets.iloc[j,1])
        tmp_comp +=  vs['compound']
        tmp_neg += vs['neg']
        tmp_neu += vs['neu']
        tmp_pos += vs['pos']

    scores.iloc[j,] = [tmp_comp, tmp_pos, tmp_neg, tmp_neu]

scores = scores.apply(pd.to_numeric)

scores.index = pd.to_datetime(scores.index)
scores['year'] = scores.index.map(lambda t: t.year)
scores['month'] = scores.index.map(lambda t: t.month)

plt.plot(scores.Compound)
plt.tight_layout()
plt.show()

scores.groupby([scores.index.map(lambda t: t.year)])['Compound'].mean().plot()
plt.show()

scores['Compound'].rolling(30).mean().plot()
plt.show()

scores['Compound'].rolling(30).std().plot()
plt.show()

import math
from textblob import TextBlob as tb

def tf(word, blob):
    return blob.words.count(word) / len(blob.words)

def n_containing(word, bloblist):
    return sum(1 for blob in bloblist if word in blob.words)

def idf(word, bloblist):
    return math.log(len(bloblist) / (1 + n_containing(word, bloblist)))

def tfidf(word, blob, bloblist):
    return tf(word, blob) * idf(word, bloblist)

docs = combined_tweets
docs.set_index('Short Date', inplace=True)


doc_2010 = docs[(docs.index >= "2010-11-01") & (docs.index < "2011-01-01")].Tweet.str.cat(sep = ' ').lower()
doc_2011 = docs[(docs.index >= "2011-11-01") & (docs.index < "2012-01-01")].Tweet.str.cat(sep = ' ').lower()
doc_2012 = docs[(docs.index >= "2012-11-01") & (docs.index < "2013-01-01")].Tweet.str.cat(sep = ' ').lower()
doc_2013 = docs[(docs.index >= "2013-11-01") & (docs.index < "2014-01-01")].Tweet.str.cat(sep = ' ').lower()
doc_2014 = docs[(docs.index >= "2014-11-01") & (docs.index < "2015-01-01")].Tweet.str.cat(sep = ' ').lower()
doc_2015 = docs[(docs.index >= "2015-11-01") & (docs.index < "2016-01-01")].Tweet.str.cat(sep = ' ').lower()
doc_2016 =docs[(docs.index >= "2016-06-01") & (docs.index < "2017-01-01")].Tweet.str.cat(sep = ' ').lower()

#bloblist = [tb(doc_2008), tb(doc_2009), tb(doc_2010), tb(doc_2011), tb(doc_2012), tb(doc_2013), tb(doc_2014)
#           , tb(doc_2015), tb(doc_2016)]

bloblist = [tb(doc_2010), tb(doc_2011), tb(doc_2012), tb(doc_2013), tb(doc_2014), tb(doc_2015), tb(doc_2016)]

for i, blob in enumerate(bloblist):
    print("Top words in document year {}".format(i + 2010))
    scores = {word: tfidf(word, blob, bloblist) for word in (set(blob.words)-stop)}
    sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)
    for word, score in sorted_words[:5]:
        print("\tWord: {}, TF-IDF: {}".format(word, round(score, 5)))