dataframe_job_based.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Importing Libraries"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Importing the dataset"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "data = pd.read_json(\"datasets/02-data/data.json\")\n",
    "data.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## creating new dataframe for the model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model_df_x = pd.DataFrame(columns=[\n",
    "    \"full name\",\n",
    "    # Skills\n",
    "    \"mobile_developer_skill\",\n",
    "    \"data_scientist_skill\",\n",
    "    \"network_engineer_skill\",\n",
    "    \"data_engineer_skill\",\n",
    "    \"software_engineer_skill\",\n",
    "    \"frontend_developer_skill\",\n",
    "    \"ui_ux_designer_skill\",\n",
    "    \"hardware_engineer_skill\",\n",
    "    \"devops_engineer_skill\",\n",
    "    \"database_administrator_skill\",\n",
    "    # Educations\n",
    "    \"bachelor-computer engineering\",\n",
    "    \"master-computer engineering\",\n",
    "    \"phd-computer engineering\",\n",
    "    \"other-majors\",\n",
    "    # Experiences\n",
    "    \"mobile developer\",\n",
    "    \"data scientist\",\n",
    "    \"ui-ux designer\",\n",
    "    \"network engineer\",\n",
    "    \"data engineer\",\n",
    "    \"software engineer\",\n",
    "    # \"full stack developer\",\n",
    "    \"frontend developer\",\n",
    "    \"hardware engineer\",\n",
    "    \"devops engineer\",\n",
    "    \"database administrator\",\n",
    "])\n",
    "model_df_x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Dataframe for the output"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model_df_y = pd.DataFrame(columns={\n",
    "    \"current experience\",\n",
    "})\n",
    "model_df_y"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Skill Buckets"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# mobile_developer = android_developer + ios_developer\n",
    "# software_engineer = software_engineer + backend_developer\n",
    "# devops engineer = devops_engineer + cloud_engineer\n",
    "# network_engineer = network_architect + network_security_engineer + penetration_teseter + wireless_network_engineer\n",
    "# data_scientist = data_scientist + machine_learning_engineer\n",
    "\"\"\"\n",
    "SKILL_BUCKETS = {\n",
    "    'mobile_developer': [\n",
    "        {\"mobile applications\", \"Mobile Application Development\"},\n",
    "        {\"java\", \"Core Java\", \"Java Enterprise Edition\", \"JavaSE\"},\n",
    "        {\"design patterns\", \"software design patterns\", },\n",
    "        {\"software engineering\", \"Software Development\", \"Software Design\"},\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "        {\"android\", \"android jetpack\", \"android data binding\", \"Android Development\", \"Android Studio\"},\n",
    "        {\"oop\", \"Object-Oriented Programming (OOP)\", \"Object Oriented Design\"},\n",
    "        {\"kotlin\"},\n",
    "        {\"xml\"},\n",
    "        {\"swift\", \"Swift (Programming Language)\"},\n",
    "        {\"ios\", \"ios development\"},\n",
    "        {\"graphic design\", \"Computer Graphics\"},\n",
    "        {\"ui-ux\", \"User Interface Design\", \"User Experience (UX)\", \"Ui Designer\", \"Ux design\",\n",
    "         \"User Interface Prototyping\", \"Material-UI\", \"User Research\"},\n",
    "        {\"sql\", \"Microsoft SQL Server\", \"SQL Server Management Studio\", \"PL/SQL\", \"T-SQL\", \"Transact-SQL (T-SQL)\", \"SSRS\", \"mysql\", \"sqlite\"},\n",
    "    ],\n",
    "    'data_scientist': [\n",
    "        {\"data migration\"},\n",
    "        {\"regression testing\"},\n",
    "        {\"predictive modeling\"},\n",
    "        {\"r\", \"R Programming\"},\n",
    "        {\"python\", \"Python (Programming Language)\"},\n",
    "        {\"image processing\"},\n",
    "        {\"mongodb\"},\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "        {\"data science\"},\n",
    "        {\"Artificial Intelligence (AI)\", \"AI\", \"Artificial Intelligence\"},\n",
    "        {\"tensorflow\"},\n",
    "        {\"pytorch\"},\n",
    "        {\"data integration\"},\n",
    "        {\"big data\"},\n",
    "        {\"NLP\"},\n",
    "        {\"data modeling\"},\n",
    "        {\"pytorch\"},\n",
    "        {\"computer vision\"},\n",
    "        {\"data\"},\n",
    "        {\"deep learning\"},\n",
    "        {\"keras\"},\n",
    "    ],\n",
    "    'network_engineer': [\n",
    "        {\"routing protocols\"},\n",
    "        {\"cisco\"},\n",
    "        {\"wireless\"},\n",
    "        {\"vpn\"},\n",
    "        {\"cybersecurity\"},\n",
    "        {\"switching\"},\n",
    "        {\"dhcp server\"},\n",
    "        {\"vmware\"},\n",
    "        {\"network security\"},\n",
    "        {\"lan-wan\"},\n",
    "        {\"hacking\"},\n",
    "        {\"cybersecurity\"},\n",
    "        {\"penetration test\"},\n",
    "        {\"network administration\"},\n",
    "        {\"linux\"},\n",
    "        {\"network security\"},\n",
    "    ],\n",
    "    'data_engineer': [\n",
    "        {\"database design\"},\n",
    "        {\"python\", \"Python (Programming Language)\"},\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "        {\"nosql\"},\n",
    "        {\"sql\", \"Microsoft SQL Server\", \"SQL Server Management Studio\", \"PL/SQL\", \"T-SQL\", \"Transact-SQL (T-SQL)\", \"SSRS\", \"mysql\", \"sqlite\"},\n",
    "        {\"postgresql\"},\n",
    "        {\"mongodb\"},\n",
    "        {\"apache kafka\"},\n",
    "        {\"graphql\"},\n",
    "        {\"spark\", \"Apache Spark\", \"PySpark\"},\n",
    "        {\"r\", \"R Programming\"},\n",
    "        {\"business analysis\"},\n",
    "        {\"data mining\"},\n",
    "    ],\n",
    "    'software_engineer': [\n",
    "        {\"java\", \"Core Java\", \"Java Enterprise Edition\", \"JavaSE\"},\n",
    "        {\"golang\", \"Go (Programming Language)\"},\n",
    "        {\"ASP.NET\", \"ASP.NET Core\", \"ASP.NET MVC\", \".NET\", \".NET Framework\", \".NET Core\", \"Asp.net identity\", \"asp.net core Web API\", \"ASP.NET WEB API\", \"LINQ\", \"Entity Framework\", \"WCF\", \"blazor\"},\n",
    "        {\"python\", \"Python (Programming Language)\"},\n",
    "        {\"django\"},\n",
    "        {\"algorithm\", \"algorithms\", \"Data Structures and algorithms\", \"data structures\", \"Graph Theory\"},\n",
    "        {\"php\"},\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "        {\"kubernetes\"},\n",
    "        {\"spring\", \"Spring Framework\", \"Spring MVC\", \"Spring Boot\"},\n",
    "        {\"docker\", \"Docker Products\"},\n",
    "        {\"laravel\"},\n",
    "        {\"mongodb\"},\n",
    "        {\"sql\", \"Microsoft SQL Server\", \"SQL Server Management Studio\", \"PL/SQL\", \"T-SQL\", \"Transact-SQL (T-SQL)\", \"SSRS\", \"mysql\", \"sqlite\"},\n",
    "        {\"javascript\", \"js\", \"JavaScript eXtension (JSX)\", \"Three.js\", \"typescript\"},\n",
    "        {\"design patterns\", \"software design patterns\", \"Software Design\"},\n",
    "        {\"web development\", \"grpc\"},\n",
    "        {\"Node.js\", \"Node\", \"Node js\"},\n",
    "        {\"database design\"},\n",
    "        {\"microservices\"},\n",
    "        {\"redis\"},\n",
    "        {\"elasticsearch\"},\n",
    "        {\"apache kafka\"},\n",
    "        {\"rabbitmq\"},\n",
    "        {\"elixir\"},\n",
    "        {\"unit testing\"},\n",
    "        {\"oop\", \"Object-Oriented Programming (OOP)\", \"Object Oriented Design\"},\n",
    "        {\"ansible\"},\n",
    "    ],\n",
    "    'frontend_developer': [\n",
    "        {\"HTML\", \"HTML5\", \"HTML 5\"},\n",
    "        {\"CSS\", \"Cascading Style Sheets (CSS)\", \"sass\", \"less\", \"Tailwind CSS\", \"scss\", \"CSS Flexbox\"},\n",
    "        {\"javascript\", \"js\", \"JavaScript eXtension (JSX)\", \"Three.js\", \"typescript\"},\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "        {\"react\", \"React.js\", \"react hook\", \"react js\", \"React Native\", \"nextjs\"},\n",
    "        {\"vuejs\", \"vue.js\"},\n",
    "        {\"angular\", \"angularjs\", \"Angular Material\", \"Angular 6\"},\n",
    "        {\"redux\", \"Redux.js\", \"Redux Thunk\"},\n",
    "        {\"ui-ux\", \"User Interface Design\", \"User Experience (UX)\", \"Ui Designer\", \"Ux design\",\n",
    "         \"User Interface Prototyping\", \"Material-UI\", \"User Research\"},\n",
    "        {\"ajax\", \"jquery\"},\n",
    "    ],\n",
    "    'ui_ux_designer': [\n",
    "        {\"graphic design\", \"Computer Graphics\"},\n",
    "        {\"illustrator\"},\n",
    "        {\"figma\", \"adobe xd\", \"Google Material Design\"},\n",
    "        {\"photoshop\", \"adobe photoshop\"},\n",
    "        {\"ui-ux\", \"User Interface Design\", \"User Experience (UX)\", \"Ui Designer\", \"Ux design\",\n",
    "         \"User Interface Prototyping\", \"Material-UI\", \"User Research\"},\n",
    "    ],\n",
    "    \"hardware_engineer\": [\n",
    "        {\"FPGA\", \"Field-Programmable Gate Arrays (FPGA)\"},\n",
    "        {\"PCB design\", \"Printed Circuit Board (PCB)\"},\n",
    "        {\"VLSI\", \"Very-Large-Scale Integration (VLSI)\"},\n",
    "        {\"IOT\"},\n",
    "        {\"circuit design\", \"Electronic Circuit Design\"},\n",
    "        {\"Computer Architecture\"},\n",
    "        {\"embedded\"},\n",
    "        {\"microcontroller\"},\n",
    "        {\"signal processing\"},\n",
    "        {\"raspberry pi\"},\n",
    "        {\"verilog\"},\n",
    "        {\"stm32\"},\n",
    "        {\"ARM\", \"ARM Assembly\"},\n",
    "        {\"xilinx\"},\n",
    "        {\"arduino\"},\n",
    "        {\"simulink\"},\n",
    "        {\"matlab\"},\n",
    "        {\"VHDL\"},\n",
    "        {\"proteus\"},\n",
    "    ],\n",
    "    \"devops_engineer\": [\n",
    "        {\"git\", \"gitflow\", \"Gitlab\", \"Github\"},\n",
    "        {\"vmware\", \"VMware Infrastructure\"},\n",
    "        {\"elasticsearch\", \"Elastic Stack (ELK)\"},\n",
    "        {\"apache kafka\", \"apache\"},\n",
    "        {\"linux\"},\n",
    "        {\"kubernetes\", \"helm chart\", \"Helm Charts\", \"openshift\", \"prometheus\"},\n",
    "        {\"docker\", \"Docker Products\"},\n",
    "        {\"ansible\"},\n",
    "        {\"infrastructure\"},#3\n",
    "        {\"google cloud platform\", \"Google Cloud Platform (GCP)\"},\n",
    "        {\"azure\", \"Azure DevOps\", \"Microsoft Azure\", \"Microsoft Azure task management\"},\n",
    "        {\"containerization\", \"Virtualization\"},\n",
    "        {\"aws\", \"amazon web services certified solutions architect\", \"Amazon CloudFront\", \"AWS Lambda\",\n",
    "         \"Amazon Redshift\", \"Amazon Web Services (AWS)\", \"AWS CloudFormation\", \"Amazon ECS\"},\n",
    "        {\"jenkins\"},\n",
    "        {\"CI-CD\", \" Continuous Delivery\", \"Continuous Integration\", \"Continuous Improvement\"},\n",
    "        {\"cloud endure\", \"cloud migration\", \"cloud backup\", \"cloud computing\"},\n",
    "        {\"devops\"},\n",
    "    ],\n",
    "    \"database_administrator\": [\n",
    "        {\"data migration\"},\n",
    "        {\"mongodb\"},\n",
    "        {\"database design\"},\n",
    "        {\"nosql\"},\n",
    "        {\"mysql\"},\n",
    "        {\"postgresql\"},\n",
    "        {\"sqlite\"},\n",
    "        {\"graphql\"},\n",
    "        {\"elasticsearch\", \"Elastic Stack (ELK)\"},\n",
    "        {\"redis\"},\n",
    "        {\"data center\"},\n",
    "        {\"mariadb\"},\n",
    "        {\"oracle\"},\n",
    "        {\"SQLite\"},\n",
    "        {\"sql\", \"Microsoft SQL Server\", \"SQL Server Management Studio\", \"PL/SQL\", \"T-SQL\", \"Transact-SQL (T-SQL)\", \"SSRS\"},\n",
    "    ],\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "outputs": [],
   "source": [
    "counted_skills = {\n",
    "    \"mobile_developer\": [False for i in range(len(SKILL_BUCKETS[\"mobile_developer\"]))],\n",
    "    \"data_scientist\": [False for i in range(len(SKILL_BUCKETS[\"data_scientist\"]))],\n",
    "    \"network_engineer\": [False for i in range(len(SKILL_BUCKETS[\"network_engineer\"]))],\n",
    "    \"data_engineer\": [False for i in range(len(SKILL_BUCKETS[\"data_engineer\"]))],\n",
    "    \"software_engineer\": [False for i in range(len(SKILL_BUCKETS[\"software_engineer\"]))],\n",
    "    \"frontend_developer\": [False for i in range(len(SKILL_BUCKETS[\"frontend_developer\"]))],\n",
    "    \"ui_ux_designer\": [False for i in range(len(SKILL_BUCKETS[\"ui_ux_designer\"]))],\n",
    "    \"hardware_engineer\": [False for i in range(len(SKILL_BUCKETS[\"hardware_engineer\"]))],\n",
    "    \"devops_engineer\": [False for i in range(len(SKILL_BUCKETS[\"devops_engineer\"]))],\n",
    "    \"database_administrator\": [False for i in range(len(SKILL_BUCKETS[\"database_administrator\"]))]\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Convert skill to lowercase"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "SKILL_MOBILE_DEVELOPER = []\n",
    "SKILL_DATA_SCIENTIST = []\n",
    "SKILL_NETWORK_ENGINEER = []\n",
    "SKILL_DATA_ENGINEER = []\n",
    "SKILL_SOFTWARE_ENGINEER = []\n",
    "SKILL_FRONTEND_DEVELOPER = []\n",
    "SKILL_UI_UX_DESIGNER = []\n",
    "SKILL_HARDWARE_ENGINEER = []\n",
    "SKILL_DEVOPS_ENGINEER = []\n",
    "SKILL_DATABASE_ADMINISTRATOR = []\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"mobile_developer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"mobile_developer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_MOBILE_DEVELOPER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"data_scientist\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"data_scientist\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DATA_SCIENTIST.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"network_engineer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"network_engineer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_NETWORK_ENGINEER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"data_engineer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"data_engineer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DATA_ENGINEER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"software_engineer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"software_engineer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_SOFTWARE_ENGINEER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"frontend_developer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"frontend_developer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_FRONTEND_DEVELOPER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"ui_ux_designer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"ui_ux_designer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_UI_UX_DESIGNER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"hardware_engineer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"hardware_engineer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_HARDWARE_ENGINEER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"devops_engineer\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"devops_engineer\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DEVOPS_ENGINEER.append(skill_set)\n",
    "\n",
    "for i in range(len(SKILL_BUCKETS[\"database_administrator\"])):\n",
    "    skill_set = set()\n",
    "    for j in SKILL_BUCKETS[\"database_administrator\"][i]:\n",
    "        skill_set.add(j.lower())\n",
    "    SKILL_DATABASE_ADMINISTRATOR.append(skill_set)\n",
    "\n",
    "\n",
    "SKILL_BUCKETS_LOWERCASE = {\n",
    "    \"mobile_developer\": SKILL_MOBILE_DEVELOPER,\n",
    "    \"data_scientist\": SKILL_DATA_SCIENTIST,\n",
    "    \"network_engineer\": SKILL_NETWORK_ENGINEER,\n",
    "    \"data_engineer\": SKILL_DATA_ENGINEER,\n",
    "    \"software_engineer\": SKILL_SOFTWARE_ENGINEER,\n",
    "    \"frontend_developer\": SKILL_FRONTEND_DEVELOPER,\n",
    "    \"ui_ux_designer\": SKILL_UI_UX_DESIGNER,\n",
    "    \"hardware_engineer\": SKILL_HARDWARE_ENGINEER,\n",
    "    \"devops_engineer\": SKILL_DEVOPS_ENGINEER,\n",
    "    \"database_administrator\": SKILL_DATABASE_ADMINISTRATOR,\n",
    "}\n",
    "\n",
    "SKILL_BUCKETS_LOWERCASE"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Education Categories"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "outputs": [],
   "source": [
    "EDUCATIONS = [\n",
    "    \"bachelor-computer engineering\",\n",
    "    \"master-computer engineering\",\n",
    "    \"phd-computer engineering\",\n",
    "    \"others\",\n",
    "]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Experience Categories"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "outputs": [],
   "source": [
    "EXPERIENCES = [\n",
    "    \"mobile developer\",  #0\n",
    "    \"data scientist\",  #1\n",
    "    \"ui-ux designer\",  #2\n",
    "    \"network engineer\",  #3\n",
    "    \"data engineer\",  #4\n",
    "    \"software engineer\",  #5\n",
    "    \"full stack developer\",  #6\n",
    "    \"frontend developer\",  #7\n",
    "    \"hardware engineer\",  #8\n",
    "    \"devops engineer\",  #9\n",
    "    \"database administrator\",  #10\n",
    "]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create conditions and append it to the dataframe"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for index, row in data.iterrows():\n",
    "    # counter for skills\n",
    "    counter_skills_mobile_developer = 0\n",
    "    counter_skills_data_scientist = 0\n",
    "    counter_skills_network_engineer = 0\n",
    "    counter_skills_data_engineer = 0\n",
    "    counter_skills_software_engineer = 0\n",
    "    counter_skills_frontend_developer = 0\n",
    "    counter_skills_ui_ux_designer = 0\n",
    "    counter_skills_hardware_engineer = 0\n",
    "    counter_skills_devops_engineer = 0\n",
    "    counter_skills_database_administrator = 0\n",
    "    # counter for educations\n",
    "    counter_educations_bachelor = 0\n",
    "    counter_educations_master = 0\n",
    "    counter_educations_phd = 0\n",
    "    counter_educations_other_majors = 0\n",
    "    # counter for experiences\n",
    "    counter_mobile_developer = 0\n",
    "    counter_data_scientist = 0\n",
    "    counter_ui_ux_designer = 0\n",
    "    counter_network_engineer = 0\n",
    "    counter_data_engineer = 0\n",
    "    counter_software_engineer = 0\n",
    "    counter_fullstack_developer = 0\n",
    "    counter_frontend_developer = 0\n",
    "    counter_hardware_engineer = 0\n",
    "    counter_devops_engineer = 0\n",
    "    counter_database_administrator = 0\n",
    "    counter_other_experiences = 0\n",
    "    current_job = \"\"\n",
    "    counted_skills = {\n",
    "        \"mobile_developer\": [False for i in range(len(SKILL_BUCKETS[\"mobile_developer\"]))],\n",
    "        \"data_scientist\": [False for i in range(len(SKILL_BUCKETS[\"data_scientist\"]))],\n",
    "        \"network_engineer\": [False for i in range(len(SKILL_BUCKETS[\"network_engineer\"]))],\n",
    "        \"data_engineer\": [False for i in range(len(SKILL_BUCKETS[\"data_engineer\"]))],\n",
    "        \"software_engineer\": [False for i in range(len(SKILL_BUCKETS[\"software_engineer\"]))],\n",
    "        \"frontend_developer\": [False for i in range(len(SKILL_BUCKETS[\"frontend_developer\"]))],\n",
    "        \"ui_ux_designer\": [False for i in range(len(SKILL_BUCKETS[\"ui_ux_designer\"]))],\n",
    "        \"hardware_engineer\": [False for i in range(len(SKILL_BUCKETS[\"hardware_engineer\"]))],\n",
    "        \"devops_engineer\": [False for i in range(len(SKILL_BUCKETS[\"devops_engineer\"]))],\n",
    "        \"database_administrator\": [False for i in range(len(SKILL_BUCKETS[\"database_administrator\"]))]\n",
    "    }\n",
    "\n",
    "    # Full name\n",
    "    name = row[\"Full Name\"]\n",
    "\n",
    "    # Skills\n",
    "    for itemSkill in row[\"Skills\"]:\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"mobile_developer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"mobile_developer\"][idx]:\n",
    "                if not counted_skills[\"mobile_developer\"][idx]:\n",
    "                    counter_skills_mobile_developer += 1\n",
    "                    counted_skills[\"mobile_developer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"data_scientist\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"data_scientist\"][idx]:\n",
    "                if not counted_skills[\"data_scientist\"][idx]:\n",
    "                    counter_skills_data_scientist += 1\n",
    "                    counted_skills[\"data_scientist\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"network_engineer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"network_engineer\"][idx]:\n",
    "                if not counted_skills[\"network_engineer\"][idx]:\n",
    "                    counter_skills_network_engineer += 1\n",
    "                    counted_skills[\"network_engineer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"data_engineer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"data_engineer\"][idx]:\n",
    "                if not counted_skills[\"data_engineer\"][idx]:\n",
    "                    counter_skills_data_engineer += 1\n",
    "                    counted_skills[\"data_engineer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"software_engineer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"software_engineer\"][idx]:\n",
    "                if not counted_skills[\"software_engineer\"][idx]:\n",
    "                    counter_skills_software_engineer += 1\n",
    "                    counted_skills[\"software_engineer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"frontend_developer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"frontend_developer\"][idx]:\n",
    "                if not counted_skills[\"frontend_developer\"][idx]:\n",
    "                    counter_skills_frontend_developer += 1\n",
    "                    counted_skills[\"frontend_developer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"ui_ux_designer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"ui_ux_designer\"][idx]:\n",
    "                if not counted_skills[\"ui_ux_designer\"][idx]:\n",
    "                    counter_skills_ui_ux_designer += 1\n",
    "                    counted_skills[\"ui_ux_designer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"hardware_engineer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"hardware_engineer\"][idx]:\n",
    "                if not counted_skills[\"hardware_engineer\"][idx]:\n",
    "                    counter_skills_hardware_engineer += 1\n",
    "                    counted_skills[\"hardware_engineer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"devops_engineer\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"devops_engineer\"][idx]:\n",
    "                if not counted_skills[\"devops_engineer\"][idx]:\n",
    "                    counter_skills_devops_engineer += 1\n",
    "                    counted_skills[\"devops_engineer\"][idx] = True\n",
    "\n",
    "        for idx, value in enumerate(SKILL_BUCKETS_LOWERCASE[\"database_administrator\"]):\n",
    "            if itemSkill.lower() in SKILL_BUCKETS_LOWERCASE[\"database_administrator\"][idx]:\n",
    "                if not counted_skills[\"database_administrator\"][idx]:\n",
    "                    counter_skills_database_administrator += 1\n",
    "                    counted_skills[\"database_administrator\"][idx] = True\n",
    "\n",
    "    # Educations\n",
    "    print(row[\"Educations\"])\n",
    "    for itemEducation in row[\"Educations\"]:\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[0]:\n",
    "            counter_educations_bachelor = 1\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[1]:\n",
    "            counter_educations_master = 1\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[2]:\n",
    "            counter_educations_phd = 1\n",
    "        if itemEducation[\"major\"] == EDUCATIONS[3]:\n",
    "            counter_educations_other_majors = 1\n",
    "\n",
    "    # Experiences\n",
    "    \"\"\"\n",
    "    Depending on the length of the experience, we build the model.\n",
    "    If the person has only 1 job, that will go to the output of the model.\n",
    "    If the person has more than one jobs, the last one is counted as model output and the rest are the input.\n",
    "    \"\"\"\n",
    "    for i in range(len(row[\"Experiences\"])):\n",
    "        if len(row[\"Experiences\"]) == 1:\n",
    "            current_job = (row[\"Experiences\"][i])[\"name\"]\n",
    "        else:\n",
    "            current_job = (row[\"Experiences\"][0])[\"name\"]\n",
    "            if i != 0:\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[0]:\n",
    "                    counter_mobile_developer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[1]:\n",
    "                    counter_data_scientist += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[2]:\n",
    "                    counter_ui_ux_designer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[3]:\n",
    "                    counter_network_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[4]:\n",
    "                    counter_data_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[5]:\n",
    "                    counter_software_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[6]:\n",
    "                    counter_frontend_developer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[7]:\n",
    "                    counter_frontend_developer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[8]:\n",
    "                    counter_hardware_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[9]:\n",
    "                    counter_devops_engineer += 1\n",
    "                if (row[\"Experiences\"][i])[\"name\"] == EXPERIENCES[10]:\n",
    "                    counter_database_administrator += 1\n",
    "\n",
    "    # Appending to the dataframe\n",
    "    model_df_x = model_df_x.append({\n",
    "        # full name\n",
    "        \"full name\": name,\n",
    "        # skills\n",
    "        'mobile_developer_skill': counter_skills_mobile_developer / len(SKILL_BUCKETS[\"mobile_developer\"]),\n",
    "        \"data_scientist_skill\": counter_skills_data_scientist / len(SKILL_BUCKETS[\"data_scientist\"]),\n",
    "        \"network_engineer_skill\": counter_skills_network_engineer / len(SKILL_BUCKETS[\"network_engineer\"]),\n",
    "        \"data_engineer_skill\": counter_skills_data_engineer / len(SKILL_BUCKETS[\"data_engineer\"]),\n",
    "        \"software_engineer_skill\": counter_skills_software_engineer / len(SKILL_BUCKETS[\"software_engineer\"]),\n",
    "        \"frontend_developer_skill\": counter_skills_frontend_developer / len(SKILL_BUCKETS[\"frontend_developer\"]),\n",
    "        \"ui_ux_designer_skill\": counter_skills_ui_ux_designer / len(SKILL_BUCKETS[\"ui_ux_designer\"]),\n",
    "        \"hardware_engineer_skill\": counter_skills_hardware_engineer / len(SKILL_BUCKETS[\"hardware_engineer\"]),\n",
    "        \"devops_engineer_skill\": counter_skills_devops_engineer / len(SKILL_BUCKETS[\"devops_engineer\"]),\n",
    "        \"database_administrator_skill\": counter_skills_database_administrator / len(SKILL_BUCKETS[\"database_administrator\"]),\n",
    "        # education\n",
    "        'bachelor-computer engineering': counter_educations_bachelor,\n",
    "        \"master-computer engineering\": counter_educations_master,\n",
    "        \"phd-computer engineering\": counter_educations_phd,\n",
    "        \"other-majors\": counter_educations_other_majors,\n",
    "        # experience\n",
    "        \"mobile developer\": counter_mobile_developer,\n",
    "        \"data scientist\": counter_data_scientist,\n",
    "        \"ui-ux designer\": counter_ui_ux_designer,\n",
    "        \"network engineer\": counter_network_engineer,\n",
    "        \"data engineer\": counter_data_engineer,\n",
    "        \"software engineer\": counter_software_engineer,\n",
    "        # \"full stack developer\": counter_fullstack_developer,\n",
    "        \"frontend developer\": counter_frontend_developer,\n",
    "        \"hardware engineer\": counter_hardware_engineer,\n",
    "        \"devops engineer\": counter_devops_engineer,\n",
    "        \"database administrator\": counter_database_administrator,\n",
    "    }, ignore_index=True)\n",
    "\n",
    "    model_df_y = model_df_y.append({\n",
    "        \"current experience\": current_job,\n",
    "    }, ignore_index=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Exporting dataframe to file"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "outputs": [],
   "source": [
    "model_df_x.to_csv('dataframe/06-skill based/df_x.csv', index=False)\n",
    "model_df_y.to_csv('dataframe/06-skill based/df_y.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## counting jobs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "job_counts = [0 for i in range(len(EXPERIENCES))]\n",
    "for idx, val in enumerate(model_df_y.values):\n",
    "    if val == EXPERIENCES[0]:\n",
    "        job_counts[0] += 1\n",
    "    elif val == EXPERIENCES[1]:\n",
    "        job_counts[1] += 1\n",
    "    elif val == EXPERIENCES[2]:\n",
    "        job_counts[2] += 1\n",
    "    elif val == EXPERIENCES[3]:\n",
    "        job_counts[3] += 1\n",
    "    elif val == EXPERIENCES[4]:\n",
    "        job_counts[4] += 1\n",
    "    elif val == EXPERIENCES[5]:\n",
    "        job_counts[5] += 1\n",
    "    elif val == EXPERIENCES[6]:\n",
    "        job_counts[6] += 1\n",
    "    elif val == EXPERIENCES[7]:\n",
    "        job_counts[7] += 1\n",
    "    elif val == EXPERIENCES[8]:\n",
    "        job_counts[8] += 1\n",
    "    elif val == EXPERIENCES[9]:\n",
    "        job_counts[9] += 1\n",
    "    elif val == EXPERIENCES[10]:\n",
    "        job_counts[10] += 1\n",
    "job_counts = np.array(job_counts)\n",
    "\n",
    "counts_of_each_job = {\n",
    "    EXPERIENCES[0]: job_counts[0],\n",
    "    EXPERIENCES[1]: job_counts[1],\n",
    "    EXPERIENCES[2]: job_counts[2],\n",
    "    EXPERIENCES[3]: job_counts[3],\n",
    "    EXPERIENCES[4]: job_counts[4],\n",
    "    EXPERIENCES[5]: job_counts[5],\n",
    "    EXPERIENCES[6]: job_counts[6],\n",
    "    EXPERIENCES[7]: job_counts[7],\n",
    "    EXPERIENCES[8]: job_counts[8],\n",
    "    EXPERIENCES[9]: job_counts[9],\n",
    "    EXPERIENCES[10]: job_counts[10],\n",
    "}\n",
    "counts_of_each_job"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}