-
Notifications
You must be signed in to change notification settings - Fork 0
/
ContractionPreprocessor.py
94 lines (82 loc) · 6 KB
/
ContractionPreprocessor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# Copyright Software Engineering Analytics Lab (SEAL), Wayne State University, 2023
# Authors: Sayma Sultana <[email protected]>, Jaydeb Sarker <[email protected]> ,and Amiangshu Bosu <[email protected]>
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# version 3 as published by the Free Software Foundation.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
import re
contraction_mapping = {"ain't": "is not", "aren't": "are not",
"can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not",
"don't": "do not", "hadn't": "had not", "hasn't": "has not",
"haven't": "have not", "he'd": "he would", "he'll": "he will",
"he's": "he is", "how'd": "how did", "how'd'y": "how do you",
"how'll": "how will", "how's": "how is", "I'd": "I would",
"I'd've": "I would have", "I'll": "I will", "I'll've": "I will have",
"I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have",
"i'll": "i will", "i'll've": "i will have", "i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have",
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have", "mightn't": "might not",
"mightn't've": "might not have", "must've": "must have",
"mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have",
"o'clock": "of the clock", "oughtn't": "ought not",
"oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have",
"she'd": "she would", "she'd've": "she would have",
"she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not",
"shouldn't've": "should not have", "so've": "so have", "so's": "so as",
"this's": "this is", "that'd": "that would", "that'd've": "that would have",
"that's": "that is", "there'd": "there would",
"there'd've": "there would have", "there's": "there is",
"here's": "here is", "they'd": "they would", "they'd've": "they would have",
"they'll": "they will", "they'll've": "they will have", "they're": "they are",
"they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not",
"what'll": "what will",
"what'll've": "what will have", "what're": "what are", "what's": "what is",
"what've": "what have", "when's": "when is", "when've": "when have",
"where'd": "where did", "where's": "where is", "where've": "where have",
"who'll": "who will", "who'll've": "who will have", "who's": "who is",
"who've": "who have", "why's": "why is", "why've": "why have",
"will've": "will have", "won't": "will not", "won't've": "will not have",
"would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would", "y'all'd've": "you all would have",
"y'all're": "you all are", "y'all've": "you all have", "you'd": "you would",
"you'd've": "you would have", "you'll": "you will", "you'll've": "you will have",
"you're": "you are", "you've": "you have", "aint": "is not", "arent": "are not",
"cant": "cannot", "cause": "because",
"couldve": "could have", "couldnt": "could not",
"didnt": "did not", "doesnt": "does not",
"dont": "do not", "hadnt": "had not", "hasnt": "has not",
"havent": "have not", "howdy": "how do you",
"its": "it is", "lets": "let us", "maam": "madam", "maynt": "may not",
"mightve": "might have", "mightnt": "might not",
"mightntve": "might not have", "mustve": "must have",
"mustnt": "must not", "mustntve": "must not have",
"neednt": "need not", "needntve": "need not have",
"oclock": "of the clock", "oughtnt": "ought not",
"shouldve": "should have", "shouldnt": "should not",
"werent": "were not", "yall": "you all", "youre": "you are",
"youve": "you have"}
url_regex = re.compile('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
def remove_url(text):
return url_regex.sub(" ", text)
def expand_contraction(text):
specials = ["’", "‘", "´", "`", "'"]
for s in specials:
text = text.replace(s, "'")
text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(" ")])
return text
def rem_special_sym(text):
# return re.sub('\W+',' ', text)
pattern = re.compile('([^\s\w]|_)+')
return pattern.sub(' ', text)