flask.py

# -*- coding: utf-8 -*-
"""LIDusingML.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1t-gfaWiuoDzb3x19cF_mg9z0_C3J25lF

# Language detection on very short sentence

#### TO-DO
1. Bi-gram visualization for freq. 2 words
2. Steamming & Lemmatization if possible
3. Grammer for each language
4. Validation
5. TF-IDF
6. ROC-AUC Graph
7. Bi-LSTM
8. Tensorflow
9. YAML file for training configuration
10. DVC
11. Docker
12. MLFlow
13. Wix (React)
14. Flask
"""

!pip install mlflow --quiet

import pandas as pd
import numpy as np
import re
import pickle
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.simplefilter("ignore")

from sklearn import metrics
from sklearn import svm
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.naive_bayes import MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

import mlflow
import mlflow.sklearn

mlflow.set_experiment(experiment_name='MLflow on Colab')

dir_path = '/content/drive/MyDrive/LID/data/'
file1 = dir_path+'data-bn.txt'
file2 = dir_path+'data-hi.txt'
file3 = dir_path+'data-ta.txt'
file4 = dir_path+'data-te.txt'
file5 = dir_path+'data-ur.txt'

fin1 = open(file1, 'r', encoding='utf8')
fin2 = open(file2, 'r', encoding='utf8')
fin3 = open(file3, 'r', encoding='utf8')
fin4 = open(file4, 'r', encoding='utf8')
fin5 = open(file5, 'r', encoding='utf8')

data=[]
lang=[]

for count,line in enumerate(fin1.readlines()):
  if count == 40165:
    break
  data.append(line.split('\t')[0])
  lang.append('bn')
fin1.close()
for count,line in enumerate(fin2.readlines()):
  if count == 40165:
    break
  data.append(line.split('\t')[0])
  lang.append('hi')
fin2.close()
for count,line in enumerate(fin3.readlines()):
  if count == 40165:
    break
  data.append(line.split('\t')[0])
  lang.append('ta')
fin3.close()
for count,line in enumerate(fin4.readlines()):
  if count == 40165:
    break
  data.append(line.split('\t')[0])
  lang.append('te')
fin4.close()
for count,line in enumerate(fin5.readlines()):
  if count == 40165:
    break
  data.append(line.split('\t')[0])
  lang.append('ur')
fin5.close()

len(data), len(lang)

data[50000]

lang[50000]

'''
Indian language constant+vowel
bn_gm = [ক,খ,গ,ঘ,ঙ,চ,ছ,জ,ঝ,ঞ,ট,ঠ,ড,ঢ,ণ,ত,থ,দ,ধ,ন,প,ফ,ব,ভ,ম,য,র,ল,শ,ষ,স,হ,য়,ড়,ঢ়]
ta_gm = [அ,ஆ,இ,ஈ,உ,ஊ,எ,ஏ,ஐ,ஒ,ஓ,ஔ,க,ங,ச,ஞ,ட,ண,த,ந,ன,ப,ம,ய,ர,ற,ல,ள,ழ,வ]
hi_gm = [क,ख,ग,घ,ङ,च,छ,ज,झ,ञ,ट,ठ,ड,ढ,ण,त,थ,द,ध,न,प,फ,ब,भ,म,य,र,ल,व,ष,स,हअ,आ,इ,ई,उ,ऊ,ए,ऐ,ओ,औ,अं,अः,ऋ,ॠ]
ur_gm = [آ,ب,,پ,ت,ٹ,ث,ج,چ,ح,خ,د,ڈ,ذ,ر,ڑ,ز,ژ,س,ش,ص,ض,ط,ظ,ع,غ,ف,ق,ک,گ,ل,م,ن,و,ہ,ی,ے]
te_gm = [అ,ఆ,ఇ,ఈ,అ,ఉ,ఊ,ఋ,ౠ,ఌ,ౡ,ఎ,ఏ,ఐ,ఒ,ఓ,ఔ,అం,అః,క,ఖ,గ,ఘ,ఙ,చ,ఛ,జ,ఝ,ఞ,ట,ఠ,డ,ఢ,ణ,త,థ,ద,ధ,న,ప,ఫ,బ,భ,మ,య,ర,ల,వ,శ,ష,స,హ,ళ,క్ష,ఱ]
'''

column = ['Data', 'Lang']
df_data = pd.DataFrame({'Data':data, 'Lang':lang})

df_data["Lang"].value_counts()

sns.countplot(y='Lang', data=df_data)

X = df_data["Data"]
y = df_data["Lang"]

"""# Label Encoder"""

le = LabelEncoder()
y = le.fit_transform(y)

"""#Text Preprocessing
#### Remove unwanted symbols, numbers
"""

# creating a list for appending the preprocessed text
data_list = []
# iterating through all the text
for text in X:
       # removing the symbols and numbers
        text = re.sub(r'[!@#$(),"%^*?:;~`0-9]', '', text)
        text = re.sub(r'[[]]', ' ', text)
        # converting the text to lower case
        text = text.lower()
        # appending to data_list
        data_list.append(text)

"""# Steaming and Lemmatization
### Root words
"""



"""#Bag of Words (Count Vectorizer)"""

cv = CountVectorizer(analyzer='char')                            #analyzer='char',ngram_range=(2, 2), min_df=1
X = cv.fit_transform(data_list) #.toarray()
X.shape # (10337, 39419)

"""# TF-IDF"""

tfidfv = TfidfVectorizer(analyzer='char')                            #analyzer='char',ngram_range=(2, 2), min_df=1
X = cv.fit_transform(data_list) #.toarray()
X.shape # (10337, 39419)

"""# Word2Vec

# Train Test Splitting
"""

x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.20)

"""# K-Fold"""

skfold = StratifiedKFold(n_splits=10, random_state=100)

"""# Model Training and Prediction

### NB
"""

skfold = StratifiedKFold(n_splits=10, random_state=100)
NBclassifier = svm.SVC()
results_skfold = cross_val_score(NBclassifier, x_train, y_train, cv=skfold)
print("Accuracy: %.2f%%" % (results_skfold.mean()*100.0))

NBclassifier = MultinomialNB()
NBclassifier.fit(x_train, y_train)

y_pred = results_skfold.predict(x_test)

ac = accuracy_score(y_test, y_pred)
cm = confusion_matrix(y_test, y_pred)

print("Accuracy is :",ac*100)

"""### SVC"""

SVMclssifier = svm.SVC()
SVMclssifier.fit(x_train, y_train, cv=skfold)

y_pred = SVMclssifier.predict(x_test)

ac = accuracy_score(y_test, y_pred)
cm = confusion_matrix(y_test, y_pred)

print("Accuracy is :",ac*100)

"""## KNN"""

KNNclssifier = KNeighborsClassifier()
KNNclssifier.fit(x_train, y_train, cv=skfold)

y_pred = KNNclssifier.predict(x_test)

ac = accuracy_score(y_test, y_pred)
cm = confusion_matrix(y_test, y_pred)

print("Accuracy is :",ac*100)

"""# Model Evaluation"""

# from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
# ac = accuracy_score(y_test, y_pred)
# cm = confusion_matrix(y_test, y_pred)

# print("Accuracy is :",ac*100)

"""# Tensorflow model"""

import tensorflow as tf
import tensorflow_hub as hub

model = "https://tfhub.dev/google/nnlm-en-dim128-with-normalization/2" #"https://tfhub.dev/google/nnlm-en-dim50/2"
hub_layer = hub.KerasLayer(model, input_shape=[], dtype=tf.string, trainable=True)
hub_layer(data_list[:3])

model = tf.keras.Sequential()
model.add(hub_layer)
model.add(tf.keras.layers.Dense(16, activation='relu'))
model.add(tf.keras.layers.Dense(5))

model.summary()

model.compile(optimizer='adam',
              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=[tf.metrics.BinaryAccuracy(threshold=0.0, name='accuracy')])

import tensorflow_datasets as tfds

x_val = data_list[:10000]
partial_x_train = data_list[10000:]

y_val = y[:10000]
partial_y_train = y[10000:]

# x_val= tf.convert_to_tensor(x_val)
# partial_x_train = tf.convert_to_tensor(partial_x_train)

# x_val = tfds.as_numpy(x_val)
# partial_x_train = tfds.as_numpy(partial_x_train)

x_val = np.array(x_val)
partial_x_train = np.array(partial_x_train)

y[0:10]

partial_x_train[:10]

x_val[:10]

history = model.fit(partial_x_train,
                    partial_y_train,
                    epochs=10,
                    batch_size=512,
                    validation_data=(x_val, y_val),
                    verbose=1)

"""# Keras model"""

input_size = 152860

model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(500,input_dim=input_size, kernel_initializer="glorot_uniform", activation="sigmoid"))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(300, kernel_initializer="glorot_uniform", activation="sigmoid"))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(100, kernel_initializer="glorot_uniform", activation="sigmoid"))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(5,kernel_initializer="glorot_uniform", activation="sigmoid"))


model.summary()

model.compile(optimizer='adam',
              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=[tf.metrics.BinaryAccuracy(threshold=0.0, name='accuracy')])

partial_x_train.shape

x_train.shape, x_test.shape, y_train.shape, y_test.shape

from keras.callbacks import TensorBoard

# Tensorboard
tensorboard = TensorBoard(log_dir="run")

# let's fit the data
# history variable will help us to plot results later
history = model.fit(x_train, y_train,
                  epochs=10,
                  validation_data=(x_test, y_test),
                  batch_size=8,
                  callbacks=[tensorboard],
                  shuffle=True)

# Evaluation on Test set
scores = model.evaluate(X_test, Y_test, verbose=1)
print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))

# and now we will prepare data for scikit-learn confusion matrix and classification report
Y_pred = model.predict_classes(X_test)
Y_pred = keras.utils.to_categorical(Y_pred, num_classes=len(LANGUAGES_DICT))
LABELS =  list(LANGUAGES_DICT.keys())

"""# MLFlow"""

def eval_metrics(actual, pred):
    rmse = np.sqrt(mean_squared_error(actual, pred))
    mae = mean_absolute_error(actual, pred)
    r2 = r2_score(actual, pred)
    return rmse, mae, r2

with mlflow.start_run():
  SVMclssifier = svm.SVC()
  SVMclssifier.fit(x_train, y_train)
  y_pred = SVMclssifier.predict(x_test)
  (rmse, mae, r2) = eval_metrics(y_test, y_pred)

  #print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio))
  print("  RMSE: %s" % rmse)
  print("  MAE: %s" % mae)
  print("  R2: %s" % r2)

  #mlflow.log_param("alpha", alpha)
  #mlflow.log_param("l1_ratio", l1_ratio)
  mlflow.log_metric("rmse", rmse)
  mlflow.log_metric("r2", r2)
  mlflow.log_metric("mae", mae)

  mlflow.sklearn.log_model(SVMclssifier, "model")

"""# Confusion matrix"""

plt.figure(figsize=(15,10))
sns.heatmap(cm, annot = True)
plt.show()

matrix = metrics.confusion_matrix(y_test, y_pred)
print(matrix)

"""# Save model in pickel"""

file = open('modelp.pkl', 'wb')
pickle.dump(model, file)
file.close()

"""# Load model"""

file2 = open('modelp.pkl', 'rb')
new_model = pickle.load(file2)
file2.close

"""# Prediction"""

def predict(text, model):
     x = cv.transform([text]).toarray() # converting text to bag of words model (Vector)
     lang = model.predict(x) # predicting the language
     if (model.predict_proba(x)[0][lang][0]) < 0.99:
       print(model.predict_proba(x)[0][lang][0])
       print('Language not detected')
     else:
       lang = le.inverse_transform(lang) # finding the language corresponding the the predicted value
       print("The langauge is in",lang[0]) # printing the language

text = "लड़के"
prep_text = re.sub(r'[!@#$(),"%^*?:;~`0-9]', '', text)
prep_text = re.sub(r'[[]]', ' ', prep_text)
prep_text = prep_text.lower()
predict(prep_text, KNNclssifier)

prep_text

"""# Visualization"""

sns.distplot(y_test-y_pred)                                  #Normal distribuation

#plt.scatter(y_test,y_pred)                                   #It shoud be linear

"""# ROC Curve"""

# y_true = []

# for i in y_pred:
#     if i==0:
#         y_true.append(1)
#     else:
#         y_true.append(0)
# y_score = []
# for i in y_test:
#     if i==0:
#         y_score.append(1)
#     else:
#         y_score.append(0)
# y_true = np.array(y_true)
# y_score = np.array(y_score)
# from sklearn.metrics import roc_curve
# from sklearn.metrics import auc
# # Compute fpr, tpr, thresholds and roc auc
# fpr, tpr, thresholds = roc_curve(y_score, y_true)
# roc_auc = auc(y_true, y_score)
# # Plot ROC curve
# plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc)
# plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
# plt.xlim([0.0, 1.0])
# plt.ylim([0.0, 1.0])
# plt.xlabel('False Positive Rate or (1 - Specifity)')
# plt.ylabel('True Positive Rate or (Sensitivity)')
# plt.title('Receiver Operating Characteristic')
# plt.legend(loc="lower right")

"""# MLFlow Visualization"""



print(mlflow.__version__)

# run tracking UI in the background
get_ipython().system_raw("mlflow ui --port 5000 &")# run tracking UI in the background

# create remote tunnel using ngrok.com to allow local port access
# borrowed from https://colab.research.google.com/github/alfozan/MLflow-GBRT-demo/blob/master/MLflow-GBRT-demo.ipynb#scrollTo=4h3bKHMYUIG6
!pip install pyngrok --quiet
from pyngrok import ngrok
from getpass import getpass
# Terminate open tunnels if exist
ngrok.kill()

# Setting the authtoken (optional)
# Get your authtoken from https://dashboard.ngrok.com/auth
NGROK_AUTH_TOKEN = ''
ngrok.set_auth_token(NGROK_AUTH_TOKEN)

# Open an HTTPs tunnel on port 5000 for http://localhost:5000
public_url = ngrok.connect(port="5000", proto="http", options={"bind_tls": True})
print("MLflow Tracking UI:", public_url)

# Commented out IPython magic to ensure Python compatibility.
# %cd /content/mlruns/
!mlflow ui