-
Notifications
You must be signed in to change notification settings - Fork 0
/
PreProcessTweets.py
92 lines (80 loc) Β· 2.65 KB
/
PreProcessTweets.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
import csv
from nltk import TweetTokenizer
from nltk.corpus import stopwords
import pickle
__author__ = 'Ritvika'
#import regex
import re
def processTweet(tweet):
# process the tweets
tweet = tweet.lower()
#Convert www.* or https?://* to URL
tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))','URL',tweet)
#Convert @username to AT_USER
tweet = re.sub('@[^\s]+','AT_USER',tweet)
#Remove additional white spaces
tweet = re.sub('[\s]+', ' ', tweet)
#Replace #word with word
tweet = re.sub(r'#([^\s]+)',r'\1', tweet)
#tweet = re.sub(r'(^[a-z]\s)|(\s[a-z]$)|(\s[a-z]\!$)|(\s[a-z]\?$)|(\s[a-z]\s)',' ',tweet)
tweet = replaceTwoOrMore(tweet)
#trim
tweet = tweet.strip('\'"')
#print tweet
tt = TweetTokenizer()
tweet_tokens = []
try:
tweet_tokens = tt.tokenize(tweet)
except UnicodeDecodeError:
#print "Error in tokenize"
pass
tweet_tokens = [ word.encode('utf-8') for word in tweet_tokens]
#print tweet_tokens
return tweet_tokens
#start process_tweet
def processTweetTrain(tweet):
tweet = tweet.lower()
tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))','URL',tweet)
tweet = re.sub('@[^\s]+','AT_USER',tweet)
tweet = re.sub('[\s]+', ' ', tweet)
tweet = re.sub(r'#([^\s]+)',r'\1', tweet)
tweet = replaceTwoOrMore(tweet)
tweet = tweet.strip('\'"')
#print tweet
tt = TweetTokenizer()
tweet_tokens = []
try:
tweet_tokens = tt.tokenize(tweet)
except UnicodeDecodeError:
#print "Error in tokenize"
pass
tweet_tokens = [ word.encode('utf-8') for word in tweet_tokens]
#print "tweet written"
#return tweet_tokens
def replaceTwoOrMore(s):
#look for 2 or more repetitions of character and replace with the character itself
pattern = re.compile(r"(.)\1{1,}", re.DOTALL)
return pattern.sub(r"\1\1", s)
#end
#Read the tweets one by one and process it
def printProcessedTweet():
"""
fp = open('sample_data.txt', 'r')
line = fp.readline()
print "printing processed tweet"
while line:
processedTweet = processTweet(line)
print processedTweet
line = fp.readline()
fp.close()
#end loop
fp = csv.reader(open('TrainingDataSA.csv', 'r'), delimiter=',')
for row in fp:
#print row
tweet = row[1]
#processedTweet = str(processTweet(tweet))
#print processedTweet
processTweetTrain(tweet)
"""
#printProcessedTweet()
#print processTweet('Yeah! So excited ! Just updated my @Flipkart for #TheBigBillionDays. https://t.co/cApJf5DCnC')