feature_extraction_user_a.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', 'c:\\\\users\\\\tanmaya\\\\appdata\\\\local\\\\programs\\\\python\\\\python35\\\\python35.zip', 'c:\\\\users\\\\tanmaya\\\\appdata\\\\local\\\\programs\\\\python\\\\python35\\\\DLLs', 'c:\\\\users\\\\tanmaya\\\\appdata\\\\local\\\\programs\\\\python\\\\python35\\\\lib', 'c:\\\\users\\\\tanmaya\\\\appdata\\\\local\\\\programs\\\\python\\\\python35', 'c:\\\\users\\\\tanmaya\\\\appdata\\\\local\\\\programs\\\\python\\\\python35\\\\lib\\\\site-packages', 'c:\\\\users\\\\tanmaya\\\\appdata\\\\local\\\\programs\\\\python\\\\python35\\\\lib\\\\site-packages\\\\IPython\\\\extensions', 'C:\\\\Users\\\\Tanmaya\\\\.ipython']\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sys\n",
    "print(sys.path)\n",
    "import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\tanmaya\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2698: DtypeWarning: Columns (8,12,13,14,15,16,50,51,52,53,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Respondent    AssessJob1    AssessJob2    AssessJob3    AssessJob4  \\\n",
      "count   98855.000000  66985.000000  66985.000000  66985.000000  66985.000000   \n",
      "mean    50822.971635      6.397089      6.673524      5.906875      4.065791   \n",
      "std     29321.650410      2.788428      2.531202      2.642734      2.541196   \n",
      "min         1.000000      1.000000      1.000000      1.000000      1.000000   \n",
      "25%     25443.500000      4.000000      5.000000      4.000000      2.000000   \n",
      "50%     50823.000000      7.000000      7.000000      6.000000      4.000000   \n",
      "75%     76219.500000      9.000000      9.000000      8.000000      6.000000   \n",
      "max    101592.000000     10.000000     10.000000     10.000000     10.000000   \n",
      "\n",
      "         AssessJob5    AssessJob6    AssessJob7    AssessJob8    AssessJob9  \\\n",
      "count  66985.000000  66985.000000  66985.000000  66985.000000  66985.000000   \n",
      "mean       3.953243      4.407196      5.673181      4.225200      7.640009   \n",
      "std        2.520499      2.502069      2.923998      2.507411      2.407457   \n",
      "min        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
      "25%        2.000000      2.000000      3.000000      2.000000      6.000000   \n",
      "50%        3.000000      4.000000      6.000000      4.000000      8.000000   \n",
      "75%        6.000000      6.000000      8.000000      6.000000     10.000000   \n",
      "max       10.000000     10.000000     10.000000     10.000000     10.000000   \n",
      "\n",
      "            ...        JobEmailPriorities6  JobEmailPriorities7  \\\n",
      "count       ...                46213.00000         46213.000000   \n",
      "mean        ...                    4.97425             4.836388   \n",
      "std         ...                    1.86063             1.659844   \n",
      "min         ...                    1.00000             1.000000   \n",
      "25%         ...                    4.00000             4.000000   \n",
      "50%         ...                    5.00000             5.000000   \n",
      "75%         ...                    7.00000             6.000000   \n",
      "max         ...                    7.00000             7.000000   \n",
      "\n",
      "       ConvertedSalary  AdsPriorities1  AdsPriorities2  AdsPriorities3  \\\n",
      "count     4.770200e+04    60479.000000    60479.000000    60479.000000   \n",
      "mean      9.578086e+04        2.726880        3.805784        3.340945   \n",
      "std       2.023482e+05        1.881078        1.821323        1.673485   \n",
      "min       0.000000e+00        1.000000        1.000000        1.000000   \n",
      "25%       2.384400e+04        1.000000        2.000000        2.000000   \n",
      "50%       5.507500e+04        2.000000        4.000000        3.000000   \n",
      "75%       9.300000e+04        4.000000        5.000000        5.000000   \n",
      "max       2.000000e+06        7.000000        7.000000        7.000000   \n",
      "\n",
      "       AdsPriorities4  AdsPriorities5  AdsPriorities6  AdsPriorities7  \n",
      "count    60479.000000    60479.000000    60479.000000    60479.000000  \n",
      "mean         3.782470        4.383604        5.138809        4.821459  \n",
      "std          1.844864        1.931746        1.853249        1.874895  \n",
      "min          1.000000        1.000000        1.000000        1.000000  \n",
      "25%          2.000000        3.000000        4.000000        3.000000  \n",
      "50%          4.000000        5.000000        6.000000        5.000000  \n",
      "75%          5.000000        6.000000        7.000000        7.000000  \n",
      "max          7.000000        7.000000        7.000000        7.000000  \n",
      "\n",
      "[8 rows x 42 columns]\n",
      "   Respondent Hobby OpenSource         Country         Student  \\\n",
      "0           1   Yes         No           Kenya              No   \n",
      "1           3   Yes        Yes  United Kingdom              No   \n",
      "2           4   Yes        Yes   United States              No   \n",
      "3           5    No         No   United States              No   \n",
      "4           7   Yes         No    South Africa  Yes, part-time   \n",
      "\n",
      "           Employment                                    FormalEducation  \\\n",
      "0  Employed part-time           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "1  Employed full-time           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "2  Employed full-time                                   Associate degree   \n",
      "3  Employed full-time           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "4  Employed full-time  Some college/university study without earning ...   \n",
      "\n",
      "                                      UndergradMajor  \\\n",
      "0                          Mathematics or statistics   \n",
      "1  A natural science (ex. biology, chemistry, phy...   \n",
      "2  Computer science, computer engineering, or sof...   \n",
      "3  Computer science, computer engineering, or sof...   \n",
      "4  Computer science, computer engineering, or sof...   \n",
      "\n",
      "                CompanySize  \\\n",
      "0        20 to 99 employees   \n",
      "1  10,000 or more employees   \n",
      "2        20 to 99 employees   \n",
      "3      100 to 499 employees   \n",
      "4  10,000 or more employees   \n",
      "\n",
      "                                             DevType      ...        \\\n",
      "0                               Full-stack developer      ...         \n",
      "1  Database administrator;DevOps specialist;Full-...      ...         \n",
      "2           Engineering manager;Full-stack developer      ...         \n",
      "3                               Full-stack developer      ...         \n",
      "4  Data or business analyst;Desktop or enterprise...      ...         \n",
      "\n",
      "                     Exercise Gender         SexualOrientation  \\\n",
      "0        3 - 4 times per week   Male  Straight or heterosexual   \n",
      "1   Daily or almost every day   Male  Straight or heterosexual   \n",
      "2                         NaN    NaN                       NaN   \n",
      "3  I don't typically exercise   Male  Straight or heterosexual   \n",
      "4        3 - 4 times per week   Male  Straight or heterosexual   \n",
      "\n",
      "                                    EducationParents  \\\n",
      "0           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "1           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
      "2                                                NaN   \n",
      "3  Some college/university study without earning ...   \n",
      "4  Some college/university study without earning ...   \n",
      "\n",
      "                  RaceEthnicity                Age Dependents  MilitaryUS  \\\n",
      "0   Black or of African descent  25 - 34 years old        Yes         NaN   \n",
      "1  White or of European descent  35 - 44 years old        Yes         NaN   \n",
      "2                           NaN                NaN        NaN         NaN   \n",
      "3  White or of European descent  35 - 44 years old         No          No   \n",
      "4  White or of European descent  18 - 24 years old        Yes         NaN   \n",
      "\n",
      "                          SurveyTooLong     SurveyEasy  \n",
      "0  The survey was an appropriate length      Very easy  \n",
      "1  The survey was an appropriate length  Somewhat easy  \n",
      "2                                   NaN            NaN  \n",
      "3  The survey was an appropriate length  Somewhat easy  \n",
      "4  The survey was an appropriate length  Somewhat easy  \n",
      "\n",
      "[5 rows x 129 columns]\n",
      "Index(['Respondent', 'Hobby', 'OpenSource', 'Country', 'Student', 'Employment',\n",
      "       'FormalEducation', 'UndergradMajor', 'CompanySize', 'DevType',\n",
      "       ...\n",
      "       'Exercise', 'Gender', 'SexualOrientation', 'EducationParents',\n",
      "       'RaceEthnicity', 'Age', 'Dependents', 'MilitaryUS', 'SurveyTooLong',\n",
      "       'SurveyEasy'],\n",
      "      dtype='object', length=129)\n",
      "Respondent                     0.000000\n",
      "Hobby                          0.000000\n",
      "OpenSource                     0.000000\n",
      "Country                        0.004168\n",
      "Student                        0.039998\n",
      "Employment                     0.035749\n",
      "FormalEducation                0.042001\n",
      "UndergradMajor                 0.200486\n",
      "CompanySize                    0.276405\n",
      "DevType                        0.068353\n",
      "YearsCoding                    0.050781\n",
      "YearsCodingProf                0.211947\n",
      "JobSatisfaction                0.299216\n",
      "CareerSatisfaction             0.226099\n",
      "HopeFiveYears                  0.234050\n",
      "JobSearchStatus                0.195913\n",
      "LastNewJob                     0.201973\n",
      "AssessJob1                     0.322391\n",
      "AssessJob2                     0.322391\n",
      "AssessJob3                     0.322391\n",
      "AssessJob4                     0.322391\n",
      "AssessJob5                     0.322391\n",
      "AssessJob6                     0.322391\n",
      "AssessJob7                     0.322391\n",
      "AssessJob8                     0.322391\n",
      "AssessJob9                     0.322391\n",
      "AssessJob10                    0.322391\n",
      "AssessBenefits1                0.343301\n",
      "AssessBenefits2                0.343301\n",
      "AssessBenefits3                0.343301\n",
      "                                 ...   \n",
      "EthicsResponsible              0.347125\n",
      "EthicalImplications            0.298882\n",
      "StackOverflowRecommend         0.226251\n",
      "StackOverflowVisit             0.222993\n",
      "StackOverflowHasAccount        0.223196\n",
      "StackOverflowParticipate       0.334986\n",
      "StackOverflowJobs              0.238309\n",
      "StackOverflowDevStory          0.335623\n",
      "StackOverflowJobsRecommend     0.612392\n",
      "StackOverflowConsiderMember    0.231126\n",
      "HypotheticalTools1             0.291204\n",
      "HypotheticalTools2             0.291366\n",
      "HypotheticalTools3             0.291902\n",
      "HypotheticalTools4             0.292034\n",
      "HypotheticalTools5             0.291437\n",
      "WakeTime                       0.270184\n",
      "HoursComputer                  0.270315\n",
      "HoursOutside                   0.271418\n",
      "SkipMeals                      0.272207\n",
      "ErgonomicDevices               0.655475\n",
      "Exercise                       0.270568\n",
      "Gender                         0.347843\n",
      "SexualOrientation              0.395428\n",
      "EducationParents               0.374710\n",
      "RaceEthnicity                  0.418613\n",
      "Age                            0.346781\n",
      "Dependents                     0.366790\n",
      "MilitaryUS                     0.840362\n",
      "SurveyTooLong                  0.332952\n",
      "SurveyEasy                     0.333579\n",
      "Length: 129, dtype: float64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Respondent', 'Hobby', 'OpenSource', 'Student', 'Employment',\n",
      "       'FormalEducation', 'UndergradMajor', 'DevType', 'YearsCoding',\n",
      "       'YearsCodingProf',\n",
      "       ...\n",
      "       'HoursOutside', 'SkipMeals', 'ErgonomicDevices', 'Exercise', 'Gender',\n",
      "       'SexualOrientation', 'EducationParents', 'Age', 'SurveyTooLong',\n",
      "       'SurveyEasy'],\n",
      "      dtype='object', length=110)\n",
      "                 Age  AssessBenefits1  AssessBenefits10  AssessBenefits11  \\\n",
      "0  25 - 34 years old              NaN               NaN               NaN   \n",
      "1  35 - 44 years old              1.0               2.0               8.0   \n",
      "2                NaN              NaN               NaN               NaN   \n",
      "3  35 - 44 years old              NaN               NaN               NaN   \n",
      "4  18 - 24 years old              1.0               9.0               6.0   \n",
      "\n",
      "   AssessBenefits2  AssessBenefits3  AssessBenefits4  AssessBenefits5  \\\n",
      "0              NaN              NaN              NaN              NaN   \n",
      "1              5.0              3.0              7.0             10.0   \n",
      "2              NaN              NaN              NaN              NaN   \n",
      "3              NaN              NaN              NaN              NaN   \n",
      "4             10.0              2.0              4.0              8.0   \n",
      "\n",
      "   AssessBenefits6  AssessBenefits7       ...         \\\n",
      "0              NaN              NaN       ...          \n",
      "1              4.0             11.0       ...          \n",
      "2              NaN              NaN       ...          \n",
      "3              NaN              NaN       ...          \n",
      "4              3.0             11.0       ...          \n",
      "\n",
      "                                     SelfTaughtTypes  \\\n",
      "0  The official documentation and/or standards fo...   \n",
      "1  The official documentation and/or standards fo...   \n",
      "2                                                NaN   \n",
      "3  The official documentation and/or standards fo...   \n",
      "4  The official documentation and/or standards fo...   \n",
      "\n",
      "          SexualOrientation             SkipMeals         Student  \\\n",
      "0  Straight or heterosexual                 Never              No   \n",
      "1  Straight or heterosexual                 Never              No   \n",
      "2                       NaN                   NaN              No   \n",
      "3  Straight or heterosexual  3 - 4 times per week              No   \n",
      "4  Straight or heterosexual                 Never  Yes, part-time   \n",
      "\n",
      "   TimeFullyProductive                                     UndergradMajor  \\\n",
      "0  One to three months                          Mathematics or statistics   \n",
      "1  One to three months  A natural science (ex. biology, chemistry, phy...   \n",
      "2                  NaN  Computer science, computer engineering, or sof...   \n",
      "3  Three to six months  Computer science, computer engineering, or sof...   \n",
      "4  Three to six months  Computer science, computer engineering, or sof...   \n",
      "\n",
      "      VersionControl                WakeTime       YearsCoding  \\\n",
      "0                Git  Between 5:00 - 6:00 AM         3-5 years   \n",
      "1     Git;Subversion  Between 6:01 - 7:00 AM  30 or more years   \n",
      "2                NaN                     NaN       24-26 years   \n",
      "3                Git  Between 6:01 - 7:00 AM       18-20 years   \n",
      "4  Zip file back-ups          Before 5:00 AM         6-8 years   \n",
      "\n",
      "   YearsCodingProf  \n",
      "0        3-5 years  \n",
      "1      18-20 years  \n",
      "2        6-8 years  \n",
      "3      12-14 years  \n",
      "4        0-2 years  \n",
      "\n",
      "[5 rows x 68 columns]\n",
      "0.29139955521\n",
      "ConvertedSalary            0.517455\n",
      "Salary                     0.488362\n",
      "FrameworkWorkedWith        0.477821\n",
      "TimeFullyProductive        0.475272\n",
      "FrameworkDesireNextYear    0.441424\n",
      "CommunicationTools         0.423701\n",
      "SelfTaughtTypes            0.419817\n",
      "DatabaseDesireNextYear     0.418360\n",
      "CurrencySymbol             0.406707\n",
      "Methodology                0.403358\n",
      "SexualOrientation          0.395428\n",
      "PlatformDesireNextYear     0.379364\n",
      "EducationParents           0.374710\n",
      "Gender                     0.347843\n",
      "Age                        0.346781\n",
      "AssessBenefits11           0.343311\n",
      "AssessBenefits2            0.343301\n",
      "AssessBenefits9            0.343301\n",
      "AssessBenefits10           0.343301\n",
      "AssessBenefits1            0.343301\n",
      "AssessBenefits7            0.343301\n",
      "AssessBenefits6            0.343301\n",
      "AssessBenefits5            0.343301\n",
      "AssessBenefits4            0.343301\n",
      "AssessBenefits3            0.343301\n",
      "AssessBenefits8            0.343301\n",
      "PlatformWorkedWith         0.332366\n",
      "DatabaseWorkedWith         0.329624\n",
      "AssessJob5                 0.322391\n",
      "AssessJob6                 0.322391\n",
      "                             ...   \n",
      "AssessJob10                0.322391\n",
      "AssessJob1                 0.322391\n",
      "EducationTypes             0.312528\n",
      "JobSatisfaction            0.299216\n",
      "EthicalImplications        0.298882\n",
      "SkipMeals                  0.272207\n",
      "HoursOutside               0.271418\n",
      "Exercise                   0.270568\n",
      "HoursComputer              0.270315\n",
      "WakeTime                   0.270184\n",
      "CheckInCode                0.268069\n",
      "LanguageDesireNextYear     0.259076\n",
      "VersionControl             0.248414\n",
      "IDE                        0.237287\n",
      "HopeFiveYears              0.234050\n",
      "OperatingSystem            0.229386\n",
      "NumberMonitors             0.227171\n",
      "CareerSatisfaction         0.226099\n",
      "YearsCodingProf            0.211947\n",
      "LanguageWorkedWith         0.207587\n",
      "LastNewJob                 0.201973\n",
      "UndergradMajor             0.200486\n",
      "DevType                    0.068353\n",
      "YearsCoding                0.050781\n",
      "FormalEducation            0.042001\n",
      "Student                    0.039998\n",
      "Employment                 0.035749\n",
      "Hobby                      0.000000\n",
      "OpenSource                 0.000000\n",
      "Respondent                 0.000000\n",
      "Length: 66, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df=pd.read_csv(\"./data/user_preprocessing/survey_results_public.csv\")\n",
    "print(df.describe())\n",
    "print(df.head())\n",
    "print(df.columns)\n",
    "\n",
    "n_row=df.shape[0]\n",
    "print(df.isnull().sum()/n_row)\n",
    "attr_to_drop=[]\n",
    "attr_to_drop.append(\"SurveyTooLong\")\n",
    "attr_to_drop.append('SurveyEasy')\n",
    "attr_to_drop.append(\"AdBlocker\")\n",
    "attr_to_drop.append(\"AdBlockerDisable\")\n",
    "attr_to_drop.append(\"AdBlockerReasons\")\n",
    "attr_to_drop.append(\"AdsAgreeDisagree1\")\n",
    "attr_to_drop.append(\"AdsAgreeDisagree2\")\n",
    "attr_to_drop.append(\"AdsAgreeDisagree3\")\n",
    "attr_to_drop.append(\"AdsActions\")\n",
    "attr_to_drop.append(\"AdsPriorities1\")\n",
    "attr_to_drop.append(\"AdsPriorities2\")\n",
    "attr_to_drop.append(\"AdsPriorities3\")\n",
    "attr_to_drop.append(\"AdsPriorities4\")\n",
    "attr_to_drop.append(\"AdsPriorities5\")\n",
    "attr_to_drop.append(\"AdsPriorities6\")\n",
    "attr_to_drop.append(\"AdsPriorities7\")\n",
    "attr_to_drop.append(\"AIDangerous\")\n",
    "attr_to_drop.append(\"AIInteresting\")\n",
    "attr_to_drop.append(\"AIResponsible\")\n",
    "attr_to_drop.append(\"AIFuture\")\n",
    "attr_to_drop.append(\"EthicsChoice\")\n",
    "attr_to_drop.append(\"EthicsReport\")\n",
    "attr_to_drop.append(\"EthicsResponsible\")\n",
    "attr_to_drop.append(\"EthicsImplications\")\n",
    "attr_to_drop.append(\"EthicsChoice\")\n",
    "attr_to_drop.append(\"StackOverflowRecommend\")\n",
    "attr_to_drop.append(\"StackOverflowVisit\")\n",
    "attr_to_drop.append(\"StackOverflowHasAccount\")\n",
    "attr_to_drop.append(\"StackOverflowParticipate\")\n",
    "attr_to_drop.append(\"StackOverflowJobs\")\n",
    "attr_to_drop.append(\"StackOverflowDevStory\")\n",
    "attr_to_drop.append(\"StackOverflowJobsRecommend\")\n",
    "attr_to_drop.append(\"StackOverflowConsiderMember\")\n",
    "attr_to_drop.append(\"HypotheticalTools1\")\n",
    "attr_to_drop.append(\"HypotheticalTools2\")\n",
    "attr_to_drop.append(\"HypotheticalTools3\")\n",
    "attr_to_drop.append(\"HypotheticalTools4\")\n",
    "attr_to_drop.append(\"HypotheticalTools5\")\n",
    "attr_to_drop.append(\"JobContactPriorities1\")\n",
    "attr_to_drop.append(\"JobContactPriorities2\")\n",
    "attr_to_drop.append(\"JobContactPriorities3\")\n",
    "attr_to_drop.append(\"JobContactPriorities4\")\n",
    "attr_to_drop.append(\"JobContactPriorities5\")\n",
    "attr_to_drop.append(\"TimeAfterBootcamp\")\n",
    "df=df.drop(df.columns[[126,3,8,15,50,51,53,62,63,64,123,125,43,44,45,46,47,48,49]],axis=1)\n",
    "print(df.columns)\n",
    "df=df[df.columns.difference(attr_to_drop)]\n",
    "print(df.head())\n",
    "attr_to_drop=[\"HackathonReasons\",\"ErgonomicDevices\"]\n",
    "df=df[df.columns.difference(attr_to_drop)]\n",
    "\n",
    "\n",
    "print(sum(df.isnull().sum())/float(np.prod(df.shape)))\n",
    "col_null=df.isnull().sum()/n_row\n",
    "print(col_null.sort_values(ascending=False))\n",
    "#print(sum(df.isnull().sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66\n",
      "54\n"
     ]
    }
   ],
   "source": [
    "#New set of attributes to be removed\n",
    "attr_to_drop=['Hobby','NumberMonitors','Salary','CheckInCode','WakeTime','TimeFullyProductive','SkipMeals','SexualOrientation','HoursOutside','Exercise','EthicalImplications','EducationParents']\n",
    "print(len(df.columns))\n",
    "df=df[df.columns.difference(attr_to_drop)]\n",
    "print(len(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df.to_csv(\"removed2.csv\")\n",
    "import numpy as np\n",
    "df1 = df.replace(np.nan, '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Xamarin', 'TensorFlow', 'Spring', 'Cordova', 'React', 'Torch/PyTorch', 'Angular', 'Node.js', 'Hadoop', 'Spark', 'Django', '.NET Core'}\n",
      "{'Xamarin', 'TensorFlow', 'Spring', 'Cordova', 'React', 'Torch/PyTorch', 'Angular', 'Node.js', 'Hadoop', 'Spark', 'Django', '.NET Core'}\n",
      "{'WordPress', 'Mac OS', 'SharePoint', 'Heroku', 'Windows Phone', 'Drupal', 'Salesforce', 'IBM Cloud or Watson', 'Serverless', 'AWS', 'Linux', 'ESP8266', 'Raspberry Pi', 'Predix', 'Android', 'Arduino', 'Firebase', 'iOS', 'Azure', 'Mainframe', 'Apple Watch or Apple TV', 'Google Home', 'Windows Desktop or Server', 'Google Cloud Platform/App Engine', 'Amazon Echo', 'Gaming console'}\n",
      "{'WordPress', 'Mac OS', 'SharePoint', 'Heroku', 'Windows Phone', 'Drupal', 'Salesforce', 'IBM Cloud or Watson', 'Serverless', 'AWS', 'Linux', 'ESP8266', 'Raspberry Pi', 'Predix', 'Android', 'Arduino', 'Firebase', 'iOS', 'Mainframe', 'Azure', 'Apple Watch or Apple TV', 'Google Home', 'Windows Desktop or Server', 'Google Cloud Platform/App Engine', 'Amazon Echo', 'Gaming console'}\n",
      "{'Python', 'C', 'Swift', 'Erlang', 'PHP', 'Assembly', 'Lua', 'Bash/Shell', 'R', 'TypeScript', 'CoffeeScript', 'Kotlin', 'Ruby', 'Scala', 'Cobol', 'Hack', 'Objective-C', 'Rust', 'SQL', 'Clojure', 'Ocaml', 'VBA', 'C++', 'HTML', 'F#', 'Groovy', 'Perl', 'JavaScript', 'Julia', 'C#', 'CSS', 'Matlab', 'Haskell', 'Go', 'Java', 'Delphi/Object Pascal', 'Visual Basic 6', 'VB.NET'}\n",
      "{'Python', 'C', 'Swift', 'Erlang', 'PHP', 'Assembly', 'Lua', 'Bash/Shell', 'R', 'TypeScript', 'CoffeeScript', 'Kotlin', 'Ruby', 'Scala', 'Cobol', 'Hack', 'Objective-C', 'Rust', 'SQL', 'Clojure', 'Ocaml', 'VBA', 'C++', 'HTML', 'F#', 'Groovy', 'Perl', 'JavaScript', 'Julia', 'C#', 'CSS', 'Matlab', 'Haskell', 'Go', 'Java', 'Delphi/Object Pascal', 'Visual Basic 6', 'VB.NET'}\n",
      "{'Slack', 'Office / productivity suite (Microsoft Office, Google Suite, etc.)', 'Jira', 'Facebook', 'Stack Overflow Enterprise', 'HipChat', 'Confluence', 'Google Hangouts/Chat', 'Other chat system (IRC, proprietary software, etc.)', 'Other wiki tool (Github, Google Sites, proprietary software, etc.)', 'Trello'}\n",
      "{'PostgreSQL', 'IBM Db2', 'Redis', 'Google BigQuery', 'Oracle', 'Amazon Redshift', 'MongoDB', 'Neo4j', 'Cassandra', 'Amazon RDS/Aurora', 'SQL Server', 'Google Cloud Storage', 'Microsoft Azure (Tables, CosmosDB, SQL, etc)', 'Apache HBase', 'MariaDB', 'Amazon DynamoDB', 'Apache Hive', 'SQLite', 'MySQL', 'Memcached', 'Elasticsearch'}\n",
      "{'PostgreSQL', 'IBM Db2', 'Redis', 'Google BigQuery', 'Oracle', 'Amazon Redshift', 'MongoDB', 'Neo4j', 'Cassandra', 'Amazon RDS/Aurora', 'SQL Server', 'Google Cloud Storage', 'Microsoft Azure (Tables, CosmosDB, SQL, etc)', 'Apache HBase', 'MariaDB', 'Amazon DynamoDB', 'Apache Hive', 'SQLite', 'MySQL', 'Memcached', 'Elasticsearch'}\n",
      "{'QA or test developer', 'Designer', 'Database administrator', 'Back-end developer', 'Data or business analyst', 'Educator or academic researcher', 'Product manager', 'Student', 'Marketing or sales professional', 'C-suite executive (CEO, CTO, etc.)', 'Mobile developer', 'Full-stack developer', 'System administrator', 'DevOps specialist', 'Front-end developer', 'Data scientist or machine learning specialist', 'Game or graphics developer', 'Embedded applications or devices developer', 'Desktop or enterprise applications developer', 'Engineering manager'}\n",
      "<class 'set'>\n",
      "['IBM Db2', 'Python', 'Back-end developer', 'WordPress', 'SharePoint', 'C', 'Redis', 'Google BigQuery', 'PHP', 'Drupal', 'Angular', 'Hadoop', 'Lua', 'Oracle', 'Data scientist or machine learning specialist', 'IBM Cloud or Watson', 'Designer', 'TypeScript', 'Jira', 'Neo4j', 'CoffeeScript', 'Kotlin', 'Cassandra', 'React', 'Stack Overflow Enterprise', 'Ruby', 'Scala', 'Amazon RDS/Aurora', 'SQL Server', 'Predix', 'Android', 'Hack', 'Educator or academic researcher', 'Rust', 'Other chat system (IRC, proprietary software, etc.)', 'Apache HBase', 'QA or test developer', 'Ocaml', 'Xamarin', 'VBA', 'C++', 'Product manager', 'Spring', 'Student', 'Marketing or sales professional', 'C-suite executive (CEO, CTO, etc.)', 'Groovy', 'JavaScript', 'Julia', 'C#', 'Desktop or enterprise applications developer', 'Office / productivity suite (Microsoft Office, Google Suite, etc.)', 'Data or business analyst', 'Haskell', 'SQLite', 'Delphi/Object Pascal', 'Spark', 'Windows Desktop or Server', 'Memcached', 'Trello', 'Elasticsearch', 'PostgreSQL', 'TensorFlow', 'Mac OS', 'Cordova', 'Swift', 'Heroku', 'Erlang', 'Full-stack developer', 'Windows Phone', 'Assembly', 'DevOps specialist', 'Amazon Redshift', 'Salesforce', 'Bash/Shell', 'R', 'Game or graphics developer', 'Engineering manager', 'Serverless', 'AWS', 'Slack', 'MongoDB', 'Visual Basic 6', 'Linux', 'ESP8266', 'HipChat', 'Cobol', 'Raspberry Pi', 'System administrator', 'Objective-C', 'Google Cloud Storage', 'Other wiki tool (Github, Google Sites, proprietary software, etc.)', 'Arduino', 'Microsoft Azure (Tables, CosmosDB, SQL, etc)', 'SQL', 'Clojure', 'MariaDB', 'Firebase', 'Database administrator', 'Facebook', 'F#', 'HTML', 'Amazon DynamoDB', 'Google Hangouts/Chat', 'iOS', 'Embedded applications or devices developer', 'Perl', 'Front-end developer', 'Confluence', 'Azure', 'Mainframe', '.NET Core', 'Apache Hive', 'CSS', 'Apple Watch or Apple TV', 'Matlab', 'Mobile developer', 'Torch/PyTorch', 'Google Home', 'Go', 'Node.js', 'Java', 'MySQL', 'Google Cloud Platform/App Engine', 'Amazon Echo', 'VB.NET', 'Django', 'Gaming console']\n"
     ]
    }
   ],
   "source": [
    "#find the unique values in each categorical attribute\n",
    "Frameworknextyear=list()\n",
    "for value in df1[\"FrameworkDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            Frameworknextyear.append(i)\n",
    "sFNY=set(Frameworknextyear)\n",
    "FrameworkWorkedWith=list()\n",
    "for value in df1[\"FrameworkWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            FrameworkWorkedWith.append(i)\n",
    "sFW=set(FrameworkWorkedWith)\n",
    "PlatformDesireNextYear=list()\n",
    "for value in df1[\"PlatformDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            PlatformDesireNextYear.append(i)\n",
    "sPDNY=set(PlatformDesireNextYear)\n",
    "PlatformWorkedWith=list()\n",
    "for value in df1[\"PlatformWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            PlatformWorkedWith.append(i)\n",
    "sPWW=set(PlatformWorkedWith)\n",
    "LanguageDesireNextYear=list()\n",
    "for value in df1[\"LanguageDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            LanguageDesireNextYear.append(i)\n",
    "sLDNY=set(LanguageDesireNextYear)\n",
    "LanguageWorkedWith=list()\n",
    "for value in df1[\"LanguageWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            LanguageWorkedWith.append(i)\n",
    "sLWW=set(LanguageWorkedWith)\n",
    "         \n",
    "CommunicationTools=list()\n",
    "for value in df1[\"CommunicationTools\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            CommunicationTools.append(i)\n",
    "sCT=set(CommunicationTools)\n",
    "\n",
    "DatabaseWorkedWith=list()\n",
    "for value in df1[\"DatabaseWorkedWith\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            DatabaseWorkedWith.append(i)\n",
    "sDWW=set(DatabaseWorkedWith)\n",
    "\n",
    "DatabaseDesireNextYear=list()\n",
    "for value in df1[\"DatabaseDesireNextYear\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            DatabaseDesireNextYear.append(i)\n",
    "sDNY=set(DatabaseDesireNextYear)\n",
    "\n",
    "DevType=list()\n",
    "for value in df1[\"DevType\"]:\n",
    "        new=value.split(';')\n",
    "        for i in new:\n",
    "            DevType.append(i)\n",
    "sDT=set(DevType)\n",
    "sFNY.remove(\"\")\n",
    "sFW.remove(\"\")\n",
    "sPDNY.remove(\"\")\n",
    "sPWW.remove(\"\")\n",
    "sLDNY.remove(\"\")\n",
    "sLWW.remove(\"\")\n",
    "sCT.remove(\"\")\n",
    "sDWW.remove(\"\")\n",
    "sDNY.remove(\"\")\n",
    "sDT.remove(\"\")\n",
    "print(sFNY)\n",
    "print(sFW)\n",
    "print(sPDNY)\n",
    "print(sPWW)\n",
    "print(sLDNY)\n",
    "print(sLWW)\n",
    "print(sCT)\n",
    "print(sDWW)\n",
    "print(sDNY)\n",
    "print(sDT)\n",
    "combined=sFNY.union(sFW,sFNY,sPDNY,sPWW,sLDNY,sLWW,sCT,sDWW,sDNY,sDT)\n",
    "print(type(combined))\n",
    "all_framework=list(combined)\n",
    "print(all_framework)\n",
    "all_fw=pd.DataFrame(all_framework)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RangeIndex(start=0, stop=1, step=1)\n"
     ]
    }
   ],
   "source": [
    "all_fw.head()\n",
    "print(all_fw.columns)\n",
    "#all_fw.to_csv(\"allFrameworks.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Neither satisfied nor dissatisfied', '', 'Slightly satisfied', 'Extremely dissatisfied', 'Moderately dissatisfied', 'Slightly dissatisfied', 'Moderately satisfied', 'Extremely satisfied'}\n"
     ]
    }
   ],
   "source": [
    "#converted categorical variables into numerical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "print(set(df1['JobSatisfaction']))\n",
    "number=LabelEncoder()\n",
    "df1['CareerSatisfaction']=number.fit_transform(df1['CareerSatisfaction'])\n",
    "df1['Employment']=number.fit_transform(df1['Employment'])\n",
    "df1['FormalEducation']=number.fit_transform(df1['FormalEducation'])\n",
    "df1['Gender']=number.fit_transform(df1['Gender'])\n",
    "df1['HopeFiveYears']=number.fit_transform(df1['HopeFiveYears'])\n",
    "df1['HoursComputer']=number.fit_transform(df1['HoursComputer'])\n",
    "df1['JobSatisfaction']=number.fit_transform(df1['JobSatisfaction'])\n",
    "df1['OpenSource']=number.fit_transform(df1['OpenSource'])\n",
    "df1['OperatingSystem']=number.fit_transform(df1['OperatingSystem'])\n",
    "df1['Student']=number.fit_transform(df1['Student'])\n",
    "df1['UndergradMajor']=number.fit_transform(df1['UndergradMajor'])\n",
    "df1['YearsCoding']=number.fit_transform(df1['YearsCoding'])\n",
    "df1['YearsCodingProf']=number.fit_transform(df1['YearsCodingProf'])\n",
    "\n",
    "#df1.to_csv(\"./data/user_preprocessing/categoricalconverted.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Salesforce\n",
      "WordPress\n",
      "Firebase\n",
      "iOS\n",
      "Android\n",
      "Predix\n",
      "Mainframe\n",
      "Amazon Echo\n",
      "Azure\n",
      "IBM Cloud or Watson\n",
      "Windows Desktop or Server\n",
      "SharePoint\n",
      "Apple Watch or Apple TV\n",
      "Google Cloud Platform/App Engine\n",
      "Gaming console\n",
      "AWS\n",
      "Mac OS\n",
      "Raspberry Pi\n",
      "Heroku\n",
      "Google Home\n",
      "Serverless\n",
      "Windows Phone\n",
      "Arduino\n",
      "Linux\n",
      "Drupal\n",
      "ESP8266\n",
      "12\n",
      "26\n"
     ]
    }
   ],
   "source": [
    "#sFNY.add('Respondent')\n",
    "#Create a separate dataframe based on one hot encoding for the attributes.\n",
    "# Framework=pd.DataFrame(columns=sFNY)\n",
    "# idx=0\n",
    "# Framework.insert(loc=idx,column='Respondent', value=[0])\n",
    "# #print(Framework.columns)\n",
    "# print(Framework.shape[0])\n",
    "# coldic=dict(zip(Framework.columns,range(0,13)))\n",
    "# print(coldic)\n",
    "# #print(df1.columns)\n",
    "# print(df1.shape[0])\n",
    "# for i in range(3):\n",
    "#     Framework = Framework.append({}, ignore_index=True)\n",
    "FrameworkWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sFW:\n",
    "    \n",
    "    FrameworkWorkedWith[i]=\"\"\n",
    "PlatformWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sPWW:\n",
    "    print(i)\n",
    "    PlatformWorkedWith[i]=\"\"\n",
    "LanguageWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sLWW:\n",
    "    LanguageWorkedWith[i]=\"\"\n",
    "DatabaseWorkedWith = pd.DataFrame(df1['Respondent'])\n",
    "for i in sDWW:\n",
    "    DatabaseWorkedWith[i]=\"\"\n",
    "DevType = pd.DataFrame(df1['Respondent'])\n",
    "for i in sDT:\n",
    "    DevType[i]=\"\"\n",
    "CommunicationTools = pd.DataFrame(df1['Respondent'])\n",
    "for i in sCT:\n",
    "    CommunicationTools[i]=\"\"\n",
    "#framework.to_csv(\"framework.csv\")\n",
    "#framework.resize=[[98855,,len(sFNY)]]\n",
    "print(len(sFW))\n",
    "print(len(sPWW))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'.NET Core': 1, 'Xamarin': 2, 'TensorFlow': 3, 'Hadoop': 4, 'Django': 5, 'Spring': 6, 'Cordova': 7, 'Node.js': 8, 'React': 9, 'Angular': 10, 'Spark': 11, 'Torch/PyTorch': 12, 'Respondent': 0}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "723.9798350334167\n",
      "{'Salesforce': 1, 'Gaming console': 15, 'AWS': 16, 'Mac OS': 17, 'Raspberry Pi': 18, 'Linux': 24, 'Google Cloud Platform/App Engine': 14, 'Heroku': 19, 'WordPress': 2, 'Firebase': 3, 'iOS': 4, 'Android': 5, 'Google Home': 20, 'Respondent': 0, 'Predix': 6, 'Mainframe': 7, 'Windows Phone': 22, 'Amazon Echo': 8, 'Drupal': 25, 'Azure': 9, 'Apple Watch or Apple TV': 13, 'Arduino': 23, 'IBM Cloud or Watson': 10, 'Serverless': 21, 'ESP8266': 26, 'Windows Desktop or Server': 11, 'SharePoint': 12}\n",
      "{'Erlang': 1, 'Ruby': 2, 'Delphi/Object Pascal': 3, 'Assembly': 4, 'C#': 22, 'CSS': 5, 'Perl': 9, 'Python': 7, 'Ocaml': 11, 'Kotlin': 10, 'Visual Basic 6': 24, 'TypeScript': 27, 'Rust': 37, 'Haskell': 25, 'CoffeeScript': 33, 'HTML': 32, 'PHP': 26, 'Java': 12, 'Bash/Shell': 16, 'VBA': 18, 'C++': 17, 'Respondent': 0, 'VB.NET': 29, 'R': 34, 'Matlab': 14, 'Objective-C': 19, 'Hack': 30, 'Clojure': 31, 'F#': 20, 'JavaScript': 6, 'Cobol': 23, 'Julia': 38, 'C': 13, 'Swift': 35, 'Lua': 21, 'SQL': 36, 'Scala': 28, 'Go': 15, 'Groovy': 8}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'JavaScript'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-78-56af40e6e2aa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     32\u001b[0m        \u001b[1;31m# print(data)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 34\u001b[1;33m             \u001b[0mLanguageWorkedWith\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcoldic\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     35\u001b[0m \u001b[1;31m#DatabasesWorkedWth\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[0mcoldic3\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDatabaseWorkedWith\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msDWW\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'JavaScript'"
     ]
    }
   ],
   "source": [
    "# print(Framework.shape[0])\n",
    "# Framework.to_csv(\"hello1.csv\")\n",
    "time_start=time.time()\n",
    "coldic=dict(zip(FrameworkWorkedWith.columns,range(0,len(sFW)+1)))\n",
    "print(coldic)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'FrameworkWorkedWith']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            FrameworkWorkedWith.iloc[i,coldic[value]]=1\n",
    "time_end=time.time()\n",
    "print(time_end-time_start)\n",
    "#PlatformWorkedWith\n",
    "coldic1=dict(zip(PlatformWorkedWith.columns,range(0,len(sPWW)+1)))\n",
    "print(coldic1)\n",
    "#print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'PlatformWorkedWith']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            PlatformWorkedWith.iloc[i,coldic1[value]]=1\n",
    "#LanguageWorkedWith\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Respondent .NET Core Xamarin TensorFlow Hadoop Django Spring Cordova  \\\n",
      "0           1                                          1                  \n",
      "1           3                                          1                  \n",
      "2           4                                                             \n",
      "3           5         1                                                   \n",
      "4           7                                                             \n",
      "\n",
      "  Node.js React Angular Spark Torch/PyTorch  \n",
      "0             1                              \n",
      "1             1                              \n",
      "2                                            \n",
      "3             1       1                      \n",
      "4                                            \n",
      "   Respondent Salesforce WordPress Firebase iOS Android Predix Mainframe  \\\n",
      "0           1                             1                                \n",
      "1           3                                                              \n",
      "2           4                                                              \n",
      "3           5                                                              \n",
      "4           7                                                              \n",
      "\n",
      "  Amazon Echo Azure   ...   Mac OS Raspberry Pi Heroku Google Home Serverless  \\\n",
      "0                 1   ...                                                       \n",
      "1                     ...                                                       \n",
      "2                     ...                                                       \n",
      "3                 1   ...                                                       \n",
      "4                     ...                                                       \n",
      "\n",
      "  Windows Phone Arduino Linux Drupal ESP8266  \n",
      "0                           1                 \n",
      "1                           1                 \n",
      "2                                             \n",
      "3                                             \n",
      "4                     1                       \n",
      "\n",
      "[5 rows x 27 columns]\n"
     ]
    }
   ],
   "source": [
    "print(FrameworkWorkedWith.head())\n",
    "print(PlatformWorkedWith.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Erlang': 1, 'Ruby': 2, 'Delphi/Object Pascal': 3, 'Assembly': 4, 'C#': 22, 'CSS': 5, 'Perl': 9, 'Python': 7, 'Ocaml': 11, 'Kotlin': 10, 'Visual Basic 6': 24, 'TypeScript': 27, 'Rust': 37, 'Haskell': 25, 'CoffeeScript': 33, 'HTML': 32, 'PHP': 26, 'Java': 12, 'Bash/Shell': 16, 'VBA': 18, 'C++': 17, 'Respondent': 0, 'VB.NET': 29, 'R': 34, 'Matlab': 14, 'Objective-C': 19, 'Hack': 30, 'Clojure': 31, 'F#': 20, 'JavaScript': 6, 'Cobol': 23, 'Julia': 38, 'C': 13, 'Swift': 35, 'Lua': 21, 'SQL': 36, 'Scala': 28, 'Go': 15, 'Groovy': 8}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "Language 3019.860470533371\n",
      "done\n",
      "{'Redis': 1, 'Oracle': 16, 'MariaDB': 2, 'Memcached': 17, 'Elasticsearch': 3, 'SQL Server': 4, 'Amazon Redshift': 18, 'Google Cloud Storage': 5, 'Apache HBase': 19, 'PostgreSQL': 6, 'Apache Hive': 7, 'Neo4j': 8, 'Respondent': 0, 'MongoDB': 9, 'IBM Db2': 10, 'SQLite': 21, 'Amazon DynamoDB': 11, 'Amazon RDS/Aurora': 12, 'Google BigQuery': 13, 'MySQL': 14, 'Microsoft Azure (Tables, CosmosDB, SQL, etc)': 20, 'Cassandra': 15}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "Databases 1180.4531762599945\n",
      "{'C-suite executive (CEO, CTO, etc.)': 13, 'Engineering manager': 1, 'Designer': 3, 'QA or test developer': 2, 'Mobile developer': 4, 'System administrator': 5, 'Game or graphics developer': 6, 'Product manager': 7, 'Data or business analyst': 8, 'Educator or academic researcher': 9, 'Marketing or sales professional': 10, 'Back-end developer': 11, 'Student': 12, 'DevOps specialist': 14, 'Front-end developer': 15, 'Database administrator': 16, 'Data scientist or machine learning specialist': 17, 'Embedded applications or devices developer': 18, 'Desktop or enterprise applications developer': 19, 'Full-stack developer': 20, 'Respondent': 0}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "DevType 1720.2005112171173\n",
      "{'Google Hangouts/Chat': 4, 'Facebook': 2, 'Other chat system (IRC, proprietary software, etc.)': 3, 'HipChat': 5, 'Jira': 1, 'Trello': 6, 'Stack Overflow Enterprise': 7, 'Other wiki tool (Github, Google Sites, proprietary software, etc.)': 8, 'Slack': 9, 'Confluence': 10, 'Office / productivity suite (Microsoft Office, Google Suite, etc.)': 11, 'Respondent': 0}\n",
      "                 Age AssessBenefits1 AssessBenefits10 AssessBenefits11  \\\n",
      "0  25 - 34 years old                                                     \n",
      "1  35 - 44 years old               1                2                8   \n",
      "2                                                                        \n",
      "3  35 - 44 years old                                                     \n",
      "4  18 - 24 years old               1                9                6   \n",
      "\n",
      "  AssessBenefits2 AssessBenefits3 AssessBenefits4 AssessBenefits5  \\\n",
      "0                                                                   \n",
      "1               5               3               7              10   \n",
      "2                                                                   \n",
      "3                                                                   \n",
      "4              10               2               4               8   \n",
      "\n",
      "  AssessBenefits6 AssessBenefits7       ...       OperatingSystem  \\\n",
      "0                                       ...                     2   \n",
      "1               4              11       ...                     2   \n",
      "2                                       ...                     0   \n",
      "3                                       ...                     4   \n",
      "4               3              11       ...                     4   \n",
      "\n",
      "              PlatformDesireNextYear                 PlatformWorkedWith  \\\n",
      "0           AWS;Azure;Linux;Firebase           AWS;Azure;Linux;Firebase   \n",
      "1                              Linux                              Linux   \n",
      "2                                                                         \n",
      "3                              Azure                              Azure   \n",
      "4  Arduino;Windows Desktop or Server  Arduino;Windows Desktop or Server   \n",
      "\n",
      "  Respondent                                    SelfTaughtTypes Student  \\\n",
      "0          1  The official documentation and/or standards fo...       1   \n",
      "1          3  The official documentation and/or standards fo...       1   \n",
      "2          4                                                          1   \n",
      "3          5  The official documentation and/or standards fo...       1   \n",
      "4          7  The official documentation and/or standards fo...       3   \n",
      "\n",
      "  UndergradMajor     VersionControl YearsCoding YearsCodingProf  \n",
      "0             11                Git           8               8  \n",
      "1              4     Git;Subversion           9               4  \n",
      "2              7                              6              10  \n",
      "3              7                Git           4               2  \n",
      "4              7  Zip file back-ups          10               1  \n",
      "\n",
      "[5 rows x 54 columns]\n"
     ]
    }
   ],
   "source": [
    "time_LWW=time.time()\n",
    "coldic2=dict(zip(LanguageWorkedWith.columns,range(0,len(sLWW)+1)))\n",
    "print(coldic2)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'LanguageWorkedWith']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            LanguageWorkedWith.iloc[i,coldic2[value]]=1\n",
    "time_LWWe=time.time()\n",
    "print(\"Language\",time_LWWe-time_LWW)\n",
    "print(\"done\")\n",
    "#DatabasesWorkedWth\n",
    "time_DWW=time.time()\n",
    "coldic3=dict(zip(DatabaseWorkedWith.columns,range(0,len(sDWW)+1)))\n",
    "print(coldic3)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'DatabaseWorkedWith']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            DatabaseWorkedWith.iloc[i,coldic3[value]]=1\n",
    "time_DWWe=time.time()\n",
    "print(\"Databases\",time_DWWe-time_DWW)\n",
    "#DevType\n",
    "#print(\"done\")\n",
    "time_DT=time.time()\n",
    "coldic4=dict(zip(DevType.columns,range(0,len(sDT)+1)))\n",
    "print(coldic4)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'DevType']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            DevType.iloc[i,coldic4[value]]=1\n",
    "time_DTe=time.time()\n",
    "print(\"DevType\",time_DTe-time_DT)\n",
    "#CommmunicationTools\n",
    "coldic5=dict(zip(CommunicationTools.columns,range(0,len(sCT)+1)))\n",
    "print(coldic5)\n",
    "print(df1.head())\n",
    "for i in range(98855):\n",
    "    data=(df1.loc[i,'CommunicationTools']).split(';')\n",
    "    if(data[0]!=\"\"):\n",
    "       # print(data)\n",
    "        for value in data:\n",
    "            CommunicationTools.iloc[i,coldic5[value]]=1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# FrameworkWorkedWith.to_csv(\"./data/user_preprocessing/FrameworkWorkedWith.csv\")\n",
    "# PlatformWorkedWith.to_csv(\"./data/user_preprocessing/PlatformWorkedWith.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# LanguageWorkedWith.to_csv(\"./data/user_preprocessing/LanguageWorkedWith.csv\")\n",
    "# DatabaseWorkedWith.to_csv(\"./data/user_preprocessing/DatabaseWorkedWith.csv\")\n",
    "# CommunicationTools.to_csv(\"./data/user_preprocessing/CommunicationTools.csv\")\n",
    "# DevType.to_csv(\"./data/user_preprocessing/DevType.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}