w2vec-model-for-autcat-problem.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install gensim --quiet","metadata":{"execution":{"iopub.status.busy":"2021-10-10T10:39:29.342399Z","iopub.execute_input":"2021-10-10T10:39:29.342844Z","iopub.status.idle":"2021-10-10T10:39:37.780660Z","shell.execute_reply.started":"2021-10-10T10:39:29.342754Z","shell.execute_reply":"2021-10-10T10:39:37.779149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gensim, gensim.downloader\nimport numpy as np\nimport re\nimport nltk\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport pickle\nimport requests\nimport os\nimport time\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-10-10T10:39:40.226333Z","iopub.execute_input":"2021-10-10T10:39:40.226729Z","iopub.status.idle":"2021-10-10T10:39:42.279909Z","shell.execute_reply.started":"2021-10-10T10:39:40.226691Z","shell.execute_reply":"2021-10-10T10:39:42.278913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading files\ncategories = list()\nlabels_all = list()\nscores = list()\nwith open('../input/hyppr-images-mapping/categories.txt', 'r') as file:\n    for line in file.readlines():\n        categories.append(line.lower().rstrip().split(','))\n        \nwith open('../input/hyppr-images-mapping/labels.txt', 'r') as file:\n    for line in file.readlines():\n        labels_all.append(line.rstrip(',\\n').split(','))\n\nwith open('../input/hyppr-images-mapping/scores.txt', 'r') as file:\n    for line in file.readlines():\n        scores.append(line.rstrip(',\\n').split(','))\n        \nwith open(\"../input/hyppr-images-mapping/all_ids.p\", \"rb\") as f:\n    all_ids = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-10-10T10:40:04.491351Z","iopub.execute_input":"2021-10-10T10:40:04.491706Z","iopub.status.idle":"2021-10-10T10:40:05.042074Z","shell.execute_reply.started":"2021-10-10T10:40:04.491677Z","shell.execute_reply":"2021-10-10T10:40:05.041015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_key(dictionary, key, value):\n    if key not in dictionary:\n         dictionary[key] = value\n    elif type(dictionary[key]) == list:\n         dictionary[key].append(value)\n    else:\n         dictionary[key] = [dictionary[key], value]","metadata":{"execution":{"iopub.status.busy":"2021-10-10T10:42:33.377500Z","iopub.execute_input":"2021-10-10T10:42:33.378015Z","iopub.status.idle":"2021-10-10T10:42:33.384651Z","shell.execute_reply.started":"2021-10-10T10:42:33.377982Z","shell.execute_reply":"2021-10-10T10:42:33.383491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_n = [\" \".join(my_list) for my_list in labels_all]\nlabels_n = [\" \".join(re.split(' |-', my_list)) for my_list in labels_n]\n\ncategories_n = [\" \".join(my_list) for my_list in categories]\ncats = [category[0] for category in categories]","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-10T10:42:37.894901Z","iopub.execute_input":"2021-10-10T10:42:37.895302Z","iopub.status.idle":"2021-10-10T10:42:38.157791Z","shell.execute_reply.started":"2021-10-10T10:42:37.895267Z","shell.execute_reply":"2021-10-10T10:42:38.156667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the tfidf vectors #\ntfidf_vec = TfidfVectorizer(ngram_range = (1, 3))\n%time tfidf_vec.fit_transform(labels_n)\n\nword_weight_dict = dict(zip(tfidf_vec.get_feature_names(), tfidf_vec.idf_))\n\nlen(tfidf_vec.get_feature_names())","metadata":{"execution":{"iopub.status.busy":"2021-10-10T10:43:36.440983Z","iopub.execute_input":"2021-10-10T10:43:36.441488Z","iopub.status.idle":"2021-10-10T10:43:40.496106Z","shell.execute_reply.started":"2021-10-10T10:43:36.441453Z","shell.execute_reply":"2021-10-10T10:43:40.495153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lexvec = gensim.models.KeyedVectors.load_word2vec_format(\n                    '../input/lexvec/lexvec.commoncrawl.300d.W.pos.neg3.vectors', binary = False)","metadata":{"execution":{"iopub.status.busy":"2021-10-10T10:43:41.862796Z","iopub.execute_input":"2021-10-10T10:43:41.863193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LEXVEC model testing","metadata":{}},{"cell_type":"code","source":"n = lexvec[0].shape[0]\n\nimages_embeddings_lv = []   # aggregating embeddings\ncount_not_working  = 0\n\nids_of_zero_vectors = [] # we need to delete zero vectors since we can't categorize it\n\nfor idimage in range(len(labels_all)):  # going on lists of words; splitting words then on subwords for aggregating subwords into one word\n    agg1 = np.zeros(n)\n    weights_sum = 0\n    for label in labels_all[idimage]:\n        agg = np.zeros(n)\n        subwords = re.split(' |--|-', label.replace('&', 'and'))\n        if len(subwords) == 1: # no <subword>-<subword>-<subword> words then\n            try:\n                agg = word_weight_dict[label] * lexvec[label]\n                weights_sum += word_weight_dict[label]\n            except KeyError as e:            \n                count_not_working += 1\n                \n        else:\n            common_weight = 0\n            for subw in subwords:    # subw for subword\n               \n                aggregate_mult = np.zeros(n)\n                if subw not in stopwords.words('english') and subw != '':  # there is some NULL strings\n                    try:\n                        aggregate_mult = word_weight_dict[subw]*lexvec[subw]\n                        common_weight += word_weight_dict[subw]\n                    except KeyError as e:\n                        count_not_working += 1\n                        \n                agg = np.add(agg, aggregate_mult)\n            if common_weight > 1e-6:\n                weights_sum += common_weight\n                agg = np.divide(agg, common_weight)\n            \n        agg1 = np.add(agg1, agg)\n    if weights_sum > 1e-6:\n        m = weights_sum\n    else:\n        m = len(labels_all[idimage])\n        ids_of_zero_vectors.append(idimage)\n        \n    images_embeddings_lv.append(np.divide(agg1, m) ) #exchange this with dictionary\n                                    \nimages_embeddings_lv = np.array(images_embeddings_lv)\nprint(count_not_working)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregating categories embeddings:","metadata":{}},{"cell_type":"code","source":"with open(\"../input/hyppr-images-mapping/category_to_urls.p\", \"rb\") as f:\n    categories_urls = pickle.load(f)\nwith open(\"../input/hyppr-images-mapping/urls_imagelabels.p\", \"rb\") as f:\n    urls_imagelabels = pickle.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = lexvec[0].shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Not taking 'dance', 'entertainment', 'tech'.","metadata":{}},{"cell_type":"code","source":"nodata = ['dance', 'entertainment', 'tech']  # we don't have data on cite for this categories so we can't consider data communicated with it\ncats = [cat for cat in cats if cat not in nodata]   # so we just throw it away","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_embeddings = dict()\nnumb_of_fails = 0\nno_url = 0\n\nfor category in cats:\n    emb_cat = np.zeros(n)\n    numb_of_images = 0\n    for image in categories_urls[category]:\n        try:\n            emb_temp = np.zeros(n)\n            distr = urls_imagelabels[image]  # try for this dictionary applying    ## there is spawning a KeyError\n            weights_sum = 0\n            numb_of_images += 1\n            for label in distr:\n                label = label[0].lower()\n                emb_temp = np.zeros(n)\n                subwords = re.split(' |--|-|, ', label)\n                \n                if len(subwords) == 1: # no <subword>-<subword>-<subword> words then\n                    try:\n                        emb_temp = word_weight_dict[label] * lexvec[label]\n                        weights_sum += word_weight_dict[label]\n                    except TypeError as e:\n                        print(e)\n                        numb_of_fails += 1\n                        \n                else:\n                    common_weight = 0\n                    for subw in subwords:    # subw for subword\n                        emb_mult = np.zeros(n)\n                        try:\n                            emb_mult = word_weight_dict[subw]*lexvec[subw]\n                            common_weight += word_weight_dict[subw]\n                        except:\n                            numb_of_fails += 1\n                        emb_temp = np.add(emb_temp, emb_mult)\n                        \n                    if common_weight > 1e-6:\n                        weights_sum += common_weight\n                        emb_temp = np.divide(emb_temp, common_weight)\n                    # there will be a lot of zero values.\n                    \n            emb_temp = np.divide(emb_temp, weights_sum)\n            emb_cat = np.add(emb_cat, emb_temp)\n        except KeyError as e:\n            no_url += 1\n    cat_embeddings[category] = np.divide(emb_cat, numb_of_images) \nprint('There is no such url: {}'.format(no_url))\nprint('Number of fails: {}'.format(numb_of_fails))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:30.181046Z","iopub.execute_input":"2021-05-25T18:42:30.181562Z","iopub.status.idle":"2021-05-25T18:42:30.255284Z","shell.execute_reply.started":"2021-05-25T18:42:30.181517Z","shell.execute_reply":"2021-05-25T18:42:30.254282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"./cat_embeddings.p\", \"wb\") as f:\n    pickle.dump(cat_embeddings, f)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:33.89032Z","iopub.execute_input":"2021-05-25T18:42:33.89081Z","iopub.status.idle":"2021-05-25T18:42:33.895756Z","shell.execute_reply.started":"2021-05-25T18:42:33.89078Z","shell.execute_reply":"2021-05-25T18:42:33.89485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing the other data which does not occur / almost does not occur on the site","metadata":{}},{"cell_type":"markdown","source":"# Quality estimation","metadata":{}},{"cell_type":"code","source":"dict_embeddings = dict()\nfor i in range(len(all_ids)):\n    dict_embeddings[all_ids[i]] = images_embeddings_lv[i]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:48.973209Z","iopub.execute_input":"2021-05-25T18:42:48.973602Z","iopub.status.idle":"2021-05-25T18:42:49.003909Z","shell.execute_reply.started":"2021-05-25T18:42:48.973573Z","shell.execute_reply":"2021-05-25T18:42:49.00304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"./dict_embeddings.p\", \"wb\") as f:\n    pickle.dump(dict_embeddings, f)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:49.156893Z","iopub.execute_input":"2021-05-25T18:42:49.157241Z","iopub.status.idle":"2021-05-25T18:42:49.62655Z","shell.execute_reply.started":"2021-05-25T18:42:49.157214Z","shell.execute_reply":"2021-05-25T18:42:49.625725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_key(dictionary, key, value):\n    if key not in dictionary:\n         dictionary[key] = value\n    elif type(dictionary[key]) == list:\n         dictionary[key].append(value)\n    else:\n         dictionary[key] = [dictionary[key], value]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:49.62787Z","iopub.execute_input":"2021-05-25T18:42:49.628174Z","iopub.status.idle":"2021-05-25T18:42:49.63301Z","shell.execute_reply.started":"2021-05-25T18:42:49.628148Z","shell.execute_reply":"2021-05-25T18:42:49.631998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open(\"../input/hyppr-images-mapping/objid_postid.p\", \"rb\") as f:\n    objid_postid = pickle.load(f)\n# reverse:\npostid_objid = dict()\nfor objid, postid in objid_postid.items():\n    set_key(postid_objid, postid, objid)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:49.634454Z","iopub.execute_input":"2021-05-25T18:42:49.634717Z","iopub.status.idle":"2021-05-25T18:42:49.753965Z","shell.execute_reply.started":"2021-05-25T18:42:49.634692Z","shell.execute_reply":"2021-05-25T18:42:49.753061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/hyppr-images-mapping/category_to_posts_vision.p', 'rb') as f:   # opening given model\n    category_to_posts = pickle.load(f)\ncategory_to_posts = dict(category_to_posts)\n\nposts_to_category = dict()\nfor cat, posts in category_to_posts.items():\n    for post in posts:\n        if post not in posts_to_category.keys():\n            set_key(posts_to_category, post, cat)\ncat_emb = list(cat_embeddings.values())","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:53.7372Z","iopub.execute_input":"2021-05-25T18:42:53.737586Z","iopub.status.idle":"2021-05-25T18:42:53.763385Z","shell.execute_reply.started":"2021-05-25T18:42:53.737556Z","shell.execute_reply":"2021-05-25T18:42:53.762524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nodata = ['dance', 'entertainment', 'tech']  # we don't have data on cite for this categories so we can't consider data communicated with it\ncats = [cat[0] for cat in categories if cat[0] not in nodata]   # so we just throw it away\ncats.append('none')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:54.058889Z","iopub.execute_input":"2021-05-25T18:42:54.059259Z","iopub.status.idle":"2021-05-25T18:42:54.064948Z","shell.execute_reply.started":"2021-05-25T18:42:54.05923Z","shell.execute_reply":"2021-05-25T18:42:54.064061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cats)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:54.631475Z","iopub.execute_input":"2021-05-25T18:42:54.631817Z","iopub.status.idle":"2021-05-25T18:42:54.636271Z","shell.execute_reply.started":"2021-05-25T18:42:54.631788Z","shell.execute_reply":"2021-05-25T18:42:54.6356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/hyppr-images-mapping/urls_imagelabels.p\", \"rb\") as f:\n    urls_imagelabels = pickle.load(f)\nwith open(\"../input/hyppr-images-mapping/postid_objurls.p\", \"rb\") as f:\n    postid_objurls = pickle.load(f)\n\nall_image_urls = []\nall_image_labels = []\nfor imageUrl, labels in urls_imagelabels.items():\n    all_image_labels.append([label[0].lower() for label in labels if label[0] != []])\n    all_image_urls.append(imageUrl)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:55.033133Z","iopub.execute_input":"2021-05-25T18:42:55.033461Z","iopub.status.idle":"2021-05-25T18:42:55.605203Z","shell.execute_reply.started":"2021-05-25T18:42:55.033432Z","shell.execute_reply":"2021-05-25T18:42:55.604236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.5\nyDef,yNew = [], []\nno_post, no_emb, no_labels = 0, 0, 0\narguable_data = dict()\nsec_dict_values = dict()\ncos_similarities = dict()\nfor post, cat in posts_to_category.items():\n    no_data = False\n    if post in postid_objid.keys():        \n        objid = postid_objid[post]\n        if objid in dict_embeddings.keys():\n            maxId = cosine_similarity(cat_emb, [dict_embeddings[objid]]).argmax()\n            if max(cosine_similarity(cat_emb, [dict_embeddings[objid]])) < threshold:    # chanching threshold for better results\n                yNew.append('none')\n            else:\n                cat_res = cats[maxId]\n                yNew.append(cat_res)\n        else:\n            no_emb += 1\n            no_data = True\n    else:\n        no_post += 1\n        no_data = True\n        \n    if no_data is False:\n        if type(cat) is list:\n            yDef.append(cat[0])\n        else:\n            yDef.append(cat)\n            \n        objid = postid_objid[post]   \n        if yNew[-1] != yDef[-1]:   # creating distributions on cosine_similarities for arguable_data\n            url = postid_objurls[post]\n            if type(url) == list:\n                url = url[0]\n            set_key(cos_similarities, cat, cosine_similarity(cat_emb, [dict_embeddings[objid]]) )\n            set_key(arguable_data, cat, url)\n            set_key(sec_dict_values, cat, yNew[-1])\n\nprint('There is {} post missing'.format(no_post))\nprint('There is {} embeddings missing'.format(no_emb))\nprint('There is {} post to labels missing'.format(no_labels))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T19:56:16.997235Z","iopub.execute_input":"2021-05-25T19:56:16.997647Z","iopub.status.idle":"2021-05-25T19:56:19.712362Z","shell.execute_reply.started":"2021-05-25T19:56:16.99761Z","shell.execute_reply":"2021-05-25T19:56:19.711308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/hyppr-images-mapping/objid_mark_category.p\", \"rb\") as f:\n       objid_mark_category = pickle.load(f)\nwith open(\"../input/hyppr-images-mapping/yRight.p\", \"rb\") as f:\n       yRight = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:42:59.818285Z","iopub.execute_input":"2021-05-25T18:42:59.818816Z","iopub.status.idle":"2021-05-25T18:42:59.845747Z","shell.execute_reply.started":"2021-05-25T18:42:59.818753Z","shell.execute_reply":"2021-05-25T18:42:59.845025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(yDef) == len(yNew)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T20:08:10.045941Z","iopub.execute_input":"2021-05-22T20:08:10.046305Z","iopub.status.idle":"2021-05-22T20:08:10.052097Z","shell.execute_reply.started":"2021-05-22T20:08:10.046264Z","shell.execute_reply":"2021-05-22T20:08:10.051001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sn\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import f1_score, precision_score, recall_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_matr = confusion_matrix(yRight, yNew, labels = cats)\n\ndf_cm = pd.DataFrame(conf_matr, index = [i for i in range(len(cats))],\n                  columns = [i for i in range(len(cats))])\nplt.figure(figsize = (10,7))\nsn.heatmap(df_cm,cmap=\"YlGnBu\",linewidths=1, annot=True, fmt = 'd')\nplt.savefig('word2vecQuality.png', bbox_inches = 'tight')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T19:56:22.993521Z","iopub.execute_input":"2021-05-25T19:56:22.99392Z","iopub.status.idle":"2021-05-25T19:56:24.571483Z","shell.execute_reply.started":"2021-05-25T19:56:22.993882Z","shell.execute_reply":"2021-05-25T19:56:24.570571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thresholds = np.arange(0,1,0.1)\nf_1_scores_new, prec_scores, rec_scores = [], [], []\nfor thresh in thresholds:\n    threshold = thresh\n    yDef,yNew = [], []\n    for post, cat in posts_to_category.items():\n        no_data = False\n        if post in postid_objid.keys():        \n            objid = postid_objid[post]\n            if objid in dict_embeddings.keys():\n                maxId = cosine_similarity(cat_emb, [dict_embeddings[objid]]).argmax()\n                if max(cosine_similarity(cat_emb, [dict_embeddings[objid]])) < threshold:    # chanching threshold for better results\n                    yNew.append('none')\n                else:\n                    cat_res = cats[maxId]\n                    yNew.append(cat_res)\n            else:\n                no_data = True\n        else:\n            no_data = True\n\n        if no_data is False:\n            if type(cat) is list:\n                yDef.append(cat[0])\n            else:\n                yDef.append(cat)\n    prec_scores.append(precision_score(yRight,yNew, labels = cats, average = 'macro'))\n    rec_scores.append(recall_score(yRight,yNew, labels = cats, average = 'macro'))\n    f_1_scores_new.append(f1_score(yRight,yNew, labels = cats, average = 'macro'))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T18:56:16.530341Z","iopub.execute_input":"2021-05-25T18:56:16.530739Z","iopub.status.idle":"2021-05-25T18:56:41.331659Z","shell.execute_reply.started":"2021-05-25T18:56:16.530698Z","shell.execute_reply":"2021-05-25T18:56:41.330471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, sharey = True, figsize = (16, 7))\nax[0].plot(thresholds, prec_scores, 'yx-')\nax[0].set_xlabel('t')\nax[0].set_ylabel('precision')\nax[1].plot(thresholds, rec_scores, 'rx-')\nax[1].set_xlabel('t')\nax[1].set_ylabel('recall')\nax[2].plot(thresholds, f_1_scores_new, 'gx-')\nax[2].set_xlabel('t')\nax[2].set_ylabel('$F_1$-score')\nplt.savefig('w2vQual.png', bbox_inches='tight')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T19:16:52.133607Z","iopub.execute_input":"2021-05-25T19:16:52.134042Z","iopub.status.idle":"2021-05-25T19:16:52.706829Z","shell.execute_reply.started":"2021-05-25T19:16:52.134008Z","shell.execute_reply":"2021-05-25T19:16:52.705824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.5\nyDef,yNew = [], []\nno_post, no_emb, no_labels = 0, 0, 0\narguable_data = dict()\nsec_dict_values = dict()\ncos_similarities = dict()\nfor post, cat in posts_to_category.items():\n    no_data = False\n    if post in postid_objid.keys():        \n        objid = postid_objid[post]\n        if objid in dict_embeddings.keys():\n            maxId = cosine_similarity(cat_emb, [dict_embeddings[objid]]).argmax()\n            if max(cosine_similarity(cat_emb, [dict_embeddings[objid]])) < threshold:    # chanching threshold for better results\n                yNew.append('none')\n            else:\n                cat_res = cats[maxId]\n                yNew.append(cat_res)\n        else:\n            no_emb += 1\n            no_data = True\n    else:\n        no_post += 1\n        no_data = True\n        \n    if no_data is False:\n        if type(cat) is list:\n            yDef.append(cat[0])\n        else:\n            yDef.append(cat)\n            \n        objid = postid_objid[post]   \n        if yNew[-1] != yDef[-1]:   # creating distributions on cosine_similarities for arguable_data\n            url = postid_objurls[post]\n            if type(url) == list:\n                url = url[0]\n            set_key(cos_similarities, cat, cosine_similarity(cat_emb, [dict_embeddings[objid]]) )\n            set_key(arguable_data, cat, url)\n            set_key(sec_dict_values, cat, yNew[-1])\n            \nconf_matr = confusion_matrix(yRight, yNew, labels = cats)\n\ndf_cm = pd.DataFrame(conf_matr, index = [i for i in range(len(cats))],\n                  columns = [i for i in range(len(cats))])\nplt.figure(figsize = (10,7))\nsn.heatmap(df_cm,cmap=\"YlGnBu\",linewidths=1, annot=True, fmt = 'd')\nplt.savefig('word2vecQuality.png', bbox_inches = 'tight')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T20:05:20.613244Z","iopub.execute_input":"2021-05-25T20:05:20.613702Z","iopub.status.idle":"2021-05-25T20:05:24.896279Z","shell.execute_reply.started":"2021-05-25T20:05:20.613664Z","shell.execute_reply":"2021-05-25T20:05:24.895093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(yRight,yNew, labels = cats, average = 'macro')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T20:05:24.898132Z","iopub.execute_input":"2021-05-25T20:05:24.898576Z","iopub.status.idle":"2021-05-25T20:05:24.93126Z","shell.execute_reply.started":"2021-05-25T20:05:24.898533Z","shell.execute_reply":"2021-05-25T20:05:24.930453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recall_score(yRight,yNew, labels = cats, average = 'macro')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T20:05:24.932809Z","iopub.execute_input":"2021-05-25T20:05:24.933097Z","iopub.status.idle":"2021-05-25T20:05:24.962746Z","shell.execute_reply.started":"2021-05-25T20:05:24.93307Z","shell.execute_reply":"2021-05-25T20:05:24.961972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"precision_score(yRight,yNew, labels = cats, average = 'macro')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T20:05:24.963975Z","iopub.execute_input":"2021-05-25T20:05:24.964242Z","iopub.status.idle":"2021-05-25T20:05:24.993466Z","shell.execute_reply.started":"2021-05-25T20:05:24.964216Z","shell.execute_reply":"2021-05-25T20:05:24.992774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final categorization of all posts:","metadata":{}},{"cell_type":"code","source":"result = dict()\nthreshold = 0.3\nfor postid, objid in postid_objid.items():        \n    objid = postid_objid[post]\n    if objid in dict_embeddings.keys():\n        maxId = cosine_similarity(cat_emb, [dict_embeddings[objid]]).argmax()\n        if max(cosine_similarity(cat_emb, [dict_embeddings[objid]])) < threshold:    # chanching threshold for better results\n            set_key(result, postid, 'none')\n        else:\n            cat_res = cats[maxId]\n            set_key(result, postid, cat_res)\n    else:\n        no_emb += 1\n        \nprint('There is {} embeddings missing'.format(no_emb))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T19:41:53.583705Z","iopub.execute_input":"2021-05-25T19:41:53.58413Z","iopub.status.idle":"2021-05-25T19:42:09.444804Z","shell.execute_reply.started":"2021-05-25T19:41:53.584093Z","shell.execute_reply":"2021-05-25T19:42:09.443632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"./postid_category.p\", \"wb\") as f:\n     pickle.dump(result, f)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T19:42:09.446627Z","iopub.execute_input":"2021-05-25T19:42:09.447051Z","iopub.status.idle":"2021-05-25T19:42:09.461323Z","shell.execute_reply.started":"2021-05-25T19:42:09.447006Z","shell.execute_reply":"2021-05-25T19:42:09.460306Z"},"trusted":true},"execution_count":null,"outputs":[]}]}