data_manipulation_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import names\n",
    "import random\n",
    "import requests\n",
    "import re\n",
    "from numpy.random import choice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureaus = ['Administration (A)',\n",
    " 'African Affairs (AF)',\n",
    " 'Allowances (A/OPR/ALS)',\n",
    " 'Arms Control, Verification and Compliance (AVC)',\n",
    " 'Authentications',\n",
    " 'Budget and Planning (BP)',\n",
    " 'Chief Information Officer (CIO)',\n",
    " 'Chief Economist, of the Department',\n",
    " 'Chief of Staff',\n",
    " 'Civil Rights, Office of',\n",
    " 'Comptroller and Global Financial Services (CGFS)',\n",
    " 'Conflict and Stabilization Operations (CSO)',\n",
    " 'Consular Affairs (CA)',\n",
    " 'Counterterrorism (CT)',\n",
    " 'Democracy, Human Rights, and Labor (DRL)',\n",
    " 'Department Spokesperson',\n",
    " 'Diplomatic Reception Rooms',\n",
    " 'Diplomatic Security (DS)',\n",
    " 'Director General of the Foreign Service and Director of Human Resources (DGHR)',\n",
    " 'East Asian and Pacific Affairs (EAP)',\n",
    " 'Economic and Business Affairs (EB)',\n",
    " 'Educational and Cultural Affairs (ECA)',\n",
    " 'Energy Resources (ENR)',\n",
    " 'European and Eurasian Affairs (EUR)',\n",
    " 'Executive Secretariat (S/ES)',\n",
    " 'Foreign Assistance (F)',\n",
    " 'Foreign Missions (OFM)',\n",
    " 'Foreign Service Institute (FSI)',\n",
    " 'Global AIDS Coordinator (S/GAC)',\n",
    " 'Global Criminal Justice (GCJ)',\n",
    " 'Global Engagement Center(GEC)',\n",
    " 'Global Food Security',\n",
    " \"Global Women's Issues (S/GWI)\",\n",
    " 'Global Youth Issues (GYI)',\n",
    " 'Human Resources (DGHR)',\n",
    " 'Information Resource Management (IRM)',\n",
    " 'Inspector General (OIG)',\n",
    " 'Intelligence and Research (INR)',\n",
    " 'International Information Programs (IIP)',\n",
    " 'International Narcotics and Law Enforcement Affairs (INL)',\n",
    " 'International Organization Affairs (IO)',\n",
    " 'International Security and Nonproliferation (ISN)',\n",
    " 'Legal Adviser (L)',\n",
    " 'Legislative Affairs (H)',\n",
    " 'Management Policy, Rightsizing and Innovation (PRI)',\n",
    " 'Medical Services (MED)',\n",
    " 'Mission to the United Nations (USUN)',\n",
    " 'Near Eastern Affairs (NEA)',\n",
    " 'Oceans and International Environmental and Scientific Affairs(OES)',\n",
    " 'Office of Terrorism Finance and Economic Sanctions Policy',\n",
    " 'Ombudsman, Office of',\n",
    " 'Overseas Buildings Operations (OBO)',\n",
    " 'Policy, Planning, and Resources for Public Diplomacy and Public Affairs (PPR)',\n",
    " 'Policy Planning Staff (S/P)',\n",
    " 'Political-Military Affairs (PM)',\n",
    " 'Population, Refugees, and Migration (PRM)',\n",
    " 'Protocol (S/CPR)',\n",
    " 'Public Affairs (PA)',\n",
    " 'Quadrennial Diplomacy and Development Review (QDDR)',\n",
    " 'Science & Technology Adviser (STAS)',\n",
    " 'South and Central Asian Affairs (SCA)',\n",
    " 'Trafficking in Persons (TIP)',\n",
    " 'Western Hemisphere Affairs (WHA)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureaus = [i for i in bureaus if '(' in i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "b_ = {}\n",
    "for i in bureaus:\n",
    "    a = i.split('(')[0]\n",
    "    a_ = i.split('(')[1]\n",
    "    b = a_.split(')')[0]\n",
    "    b_[a.strip()] = b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('bureaus.json','w') as f:\n",
    "    f.write(json.dumps(b_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "skills = open('linkedin_skills.txt','r').readlines()\n",
    "skills = [i.split('\\n')[0] for i in skills]\n",
    "\n",
    "# remove any skills that contain things that are not alphanumeric\n",
    "skills = [i for i in skills if bool(re.match('^[a-zA-Z0-9]+$', i))]\n",
    "\n",
    "# lets use this more relevant list of skills\n",
    "skills = ['Excel',\n",
    "          'Powerpoint',\n",
    "          'Communications',\n",
    "          'Email',\n",
    "          'Foreign Policy',\n",
    "          'Engineering',\n",
    "          'Hydrology',\n",
    "          'Disaster Response',\n",
    "          'Climatology',\n",
    "          'Economics',\n",
    "          'Healthcare',\n",
    "          'Outlook',\n",
    "          'Design',\n",
    "          'Graphics',\n",
    "          'Visualization',\n",
    "          'Spanish',\n",
    "          'Chinese',\n",
    "          'Japanese',\n",
    "          'French',\n",
    "          'Portuguese',\n",
    "          'German',\n",
    "          'Finance',\n",
    "          'Management',\n",
    "          'Research',\n",
    "          'Planning',\n",
    "          'Operations',\n",
    "          'Budget',\n",
    "          'Writing',\n",
    "          'Editing',\n",
    "          'Sharepoint',\n",
    "          'Social Media',\n",
    "          'Marketing',\n",
    "          'Tourism',\n",
    "          'Government',\n",
    "          'Disease Response',\n",
    "          'Interagency',\n",
    "          'Data Analysis',\n",
    "          'IT',\n",
    "          'Cloud Computing',\n",
    "          'Human Resources',\n",
    "          'Education',\n",
    "          'Web Development',\n",
    "          'SEO',\n",
    "          'Network Security',\n",
    "          'Leadership',\n",
    "          'Training',\n",
    "          'Administration'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "draw = choice(skills, 5, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Disease Response', 'Network Security', 'Communications',\n",
       "       'Operations', 'Graphics'], dtype='<U17')"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_person():\n",
    "    # random variables\n",
    "    random_cat = random.random()\n",
    "    random_senior = random.random()\n",
    "    random_ws_trail = random.random()\n",
    "    random_ws_start = random.random()    \n",
    "    random_last_only = random.random()\n",
    "    random_grade_nodash = random.random()\n",
    "    random_grade_space = random.random()\n",
    "\n",
    "    # select Gender\n",
    "    Gender = random.choice(['male','female'])\n",
    "    \n",
    "    # select Name\n",
    "    if random_last_only > 0.9:\n",
    "        Name = names.get_last_name()\n",
    "    else:\n",
    "        Name = names.get_full_name(gender=Gender)\n",
    "        \n",
    "    # add random trailing space to Name\n",
    "    if random_ws_trail > 0.7:\n",
    "        num_spaces = random.randint(1,5)\n",
    "        for i in range(num_spaces):\n",
    "            Name += ' '\n",
    "    \n",
    "    # add random leading space to Name\n",
    "    if random_ws_start > 0.7:\n",
    "        num_spaces = random.randint(1,5)\n",
    "        for i in range(num_spaces):\n",
    "            Name = ' ' + Name\n",
    "    \n",
    "    # select Bureau\n",
    "    Bureau = random.choice(bureaus)\n",
    "    \n",
    "    # generate skills\n",
    "    Skills = []\n",
    "    skills_num = random.randint(0, 12)\n",
    "    if skills_num > 0:\n",
    "        for i in range(skills_num):\n",
    "            sk = list(choice(skills, skills_num, replace=False))\n",
    "        for i in sk:\n",
    "            if random.random() > 0.75:\n",
    "                i = i.lower()\n",
    "            Skills.append(i)\n",
    "    \n",
    "    # generate years in service\n",
    "    YearsService = random.randint(1,25)\n",
    "    \n",
    "    # generate category/grade\n",
    "    if random_cat < 0.3:\n",
    "        Category = 'Contractor'\n",
    "        Grade = 'N/A'\n",
    "    elif random_cat < 0.6:\n",
    "        Category = 'Civil Service'\n",
    "        if random_senior > 0.95:\n",
    "            gs = random.randint(1,5)\n",
    "            Grade = 'SES-' + str(gs)\n",
    "            YearsService = random.randint(18,25)\n",
    "        else:\n",
    "            gs = random.randint(7,15)\n",
    "            Grade = 'GS-' + str(gs)\n",
    "    else:\n",
    "        Category = 'Foreign Service'\n",
    "        if random_senior > 0.95:\n",
    "            fs = random.randint(1,5)\n",
    "            Grade = 'SFS-' + str(fs)\n",
    "            YearsService = random.randint(18,25)\n",
    "        else:\n",
    "            fs = random.randint(1,9)\n",
    "            Grade = 'FS-' + str(fs)\n",
    "    \n",
    "    # Remove dash from Grade\n",
    "    if Grade != 'N/A':\n",
    "        if random_grade_nodash > 0.9:\n",
    "            Grade = Grade.replace('-','')\n",
    "        elif random_grade_space > 0.9:\n",
    "            Grade = Grade.replace('-',' ')\n",
    "    \n",
    "    EE = {'Gender': Gender,\n",
    "          'Name': Name,\n",
    "          'Bureau': Bureau,\n",
    "          'Skills': \", \".join(Skills),\n",
    "          'YearsService': YearsService,\n",
    "          'Category': Category,\n",
    "          'Grade': Grade}\n",
    "\n",
    "    return EE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "cont = False\n",
    "while cont == False:\n",
    "    check = gen_person()\n",
    "    if check['Category'] != 'Contractor':\n",
    "        if ' ' in check['Grade']:\n",
    "            cont = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "EEs = []\n",
    "for i in range(10000):\n",
    "    EEs.append(gen_person())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Bureau': 'Comptroller and Global Financial Services (CGFS)',\n",
       " 'Category': 'Contractor',\n",
       " 'Gender': 'female',\n",
       " 'Grade': 'N/A',\n",
       " 'Name': '    Katrina Lilly     ',\n",
       " 'Skills': 'Hydrology, Research, Design, human resources, Administration',\n",
       " 'YearsService': 16}"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EEs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns=['Name','Gender','Grade','Bureau','Skills','Category','YearsService'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.from_records(EEs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ = df[['Bureau','Gender','Grade','Name','Skills','YearsService']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_.to_csv('HR_Data.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}