spanglish_playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src import parse_miami\n",
    "from src import cm_metrics\n",
    "from src import calc_cocoa as cc\n",
    "from src import new_metric_strat as nms\n",
    "from collections import defaultdict\n",
    "# from src import calc_cm_strategies as ccs\n",
    "\n",
    "mi_utt_file = './data/all_cm/miami_all_ids.0211'  # miami_all.0211\n",
    "\n",
    "data_folder_path = \"./data/miami/clean_1208/\"\n",
    "miami_data = parse_miami.load_data(data_folder_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['uhuh_eng'], [], ['ya_spa', 'estamos_spa', 'dentro_spa', 'que_spa']]\n",
      "['mi_zeledon8_0002_FLA', 'mi_zeledon8_0009_FLA', 'mi_zeledon8_0010_FLA']\n",
      "[[1], [], [0, 0, 0, 0]]\n"
     ]
    }
   ],
   "source": [
    "print miami_data['zeledon8']['FLA']['words'][:3]\n",
    "print miami_data['zeledon8']['FLA']['uttid'][:3]\n",
    "print miami_data['zeledon8']['FLA']['words_01'][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(cm_metrics)\n",
    "txt_dict, strat_dict = cm_metrics.get_style_metrics_miami(miami_data)\n",
    "# # print miami_dict.keys()  # ['neither', 's-0', 's-1', 'c-1', 'c-0']\n",
    "# print len(strat_dict.keys())  # 2889"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80\n",
      "['BRA', 'EVN', 'JES', 'LIL', 'DIE', 'TOM', 'EMI', 'EVE', 'MIG', 'CON', 'TIM', 'NIC', 'PAI', 'NAT', 'AAR', 'FLA', 'LUI', 'ISA', 'LUK', 'ETH', 'PAO', 'CAL', 'ARI', 'JAD', 'HER', 'SAM', 'SOF', 'OLI', 'ROB', 'AME', 'JUL', 'JOC', 'ELL', 'SEB', 'SEA', 'CAR', 'SAR', 'ASH', 'GAB', 'GRA', 'CHA', 'ELE', 'IAN', 'VAN', 'AUD', 'VAL', 'VIC', 'ABI', 'FEL', 'CHL', 'JAS', 'CLA', 'IRI', 'JAM', 'MEG', 'GIA', 'LAU', 'LAN', 'HEN', 'GIL', 'JEN', 'JAC', 'REB', 'LEA', 'MAS', 'MAR', 'MAT', 'MAY', 'MEL', 'ABE', 'AVE', 'NOA', 'RIC', 'ANT', 'MAD', 'ALY', 'KEV', 'KAY', 'AVA', 'LAR']\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# print strat_dict.keys()[0] # mi_zeledon7_0809_JAS\n",
    "spkr_csid = defaultdict(list)\n",
    "for csid in strat_dict:\n",
    "    spkr_csid[csid.split('_')[3]].append(csid)\n",
    "print len(spkr_csid) # 80\n",
    "print spkr_csid.keys()\n",
    "print 'ELI' in spkr_csid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "not CS COL\n",
      "not CS ELI\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "socio_map = defaultdict(dict)\n",
    "lang_spkr_map = {}\n",
    "# ex: socio_map['IRI'] = {'PR': 'PR', 'sp': 4, 'en': 4}\n",
    "nat_map = defaultdict(list)\n",
    "lang_map = defaultdict(list)\n",
    "with open('data/miami/miami_spkrs_socio_121019.csv') as csvfile:\n",
    "    reader = csv.DictReader(csvfile)\n",
    "    for row in reader:\n",
    "        spkr = row['spkr']\n",
    "        if spkr not in spkr_csid:\n",
    "            print 'not CS', spkr\n",
    "        socio_map[spkr]['nation'] = row['nation']\n",
    "        spa_abil = int(row['spa_abil'])\n",
    "        en_abil = int(row['en_abil'])\n",
    "        socio_map[spkr]['sp'] = spa_abil\n",
    "        socio_map[spkr]['en'] = en_abil\n",
    "        nat_map[row['nation']].append(spkr)\n",
    "        \n",
    "        if spa_abil == en_abil:\n",
    "            lang_key = 'sym'\n",
    "        elif spa_abil > en_abil:\n",
    "            lang_key = 'dom_spa'\n",
    "        else:\n",
    "            lang_key = 'dom_eng'\n",
    "        lang_map[lang_key].append(spkr)\n",
    "        lang_spkr_map[spkr] = lang_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dom_spa': 8, 'sym': 61, 'dom_eng': 14}\n"
     ]
    }
   ],
   "source": [
    "print {key: len(val) for key, val in lang_map.iteritems()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get nat of spkrs >= 3\n",
    "nat_hifreq = {}\n",
    "for nat in sorted(nat_map.keys()):\n",
    "    if len(nat_map[nat]) > 2:\n",
    "        nat_hifreq[nat] = nat_map[nat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PR 3 Counter({'dom_spa': 1, 'dom_eng': 1, 'sym': 1})\n",
      "Dominican 3 Counter({'sym': 2, 'dom_eng': 1})\n",
      "Cuba 18 Counter({'sym': 12, 'dom_spa': 4, 'dom_eng': 2})\n",
      "Venezuela 3 Counter({'sym': 3})\n",
      "US 23 Counter({'sym': 15, 'dom_eng': 7, 'dom_spa': 1})\n",
      "5 5 Counter({'sym': 4, 'dom_spa': 1})\n",
      "Colombia 7 Counter({'sym': 7})\n",
      "US/Cuba 8 Counter({'sym': 7, 'dom_eng': 1})\n",
      "Nicaragua 4 Counter({'sym': 3, 'dom_spa': 1})\n",
      "Ecuador 3 Counter({'sym': 3})\n",
      "77\n"
     ]
    }
   ],
   "source": [
    "for nat, spkrs in nat_hifreq.iteritems():\n",
    "    print nat, len(spkrs), Counter([lang_spkr_map[spkr] for spkr in spkrs])\n",
    "print sum([len(spkrs) for spkrs in nat_hifreq.values()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPKR not present ELI\n"
     ]
    }
   ],
   "source": [
    "# from collections import Counter\n",
    "strat_ratio = {}\n",
    "coarse_strat_ratio = {}\n",
    "spkr_numutts = {}\n",
    "for nat, spkrs in nat_hifreq.iteritems():\n",
    "    for spkr in spkrs:\n",
    "        if spkr not in spkr_csid:\n",
    "            print 'SPKR not present', spkr\n",
    "            continue\n",
    "        csids = spkr_csid[spkr]\n",
    "        ct_strat = Counter([strat_dict[csid] for csid in csids])\n",
    "        coarse_ct_strat = {'n': ct_strat['neither'], 's': ct_strat['s-1'] + ct_strat['s-0'], 'c': ct_strat['c-1'] + ct_strat['c-0']}\n",
    "#         print spkr, coarse_ct_strat\n",
    "        sum_utt = sum(ct_strat.values())\n",
    "        strat_ratio[spkr] = {key: float(val)/sum_utt for key, val in ct_strat.iteritems()}\n",
    "        coarse_strat_ratio[spkr] = {key: float(val)/sum_utt for key, val in coarse_ct_strat.iteritems()}\n",
    "        spkr_numutts[spkr] = sum_utt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'neither': 0.3488372093023256, 's-1': 0.16279069767441862, 'c-1': 0.023255813953488372, 'c-0': 0.46511627906976744}\n",
      "513\n"
     ]
    }
   ],
   "source": [
    "print strat_ratio['IRI']\n",
    "print spkr_numutts['MAR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Other\tneither\t0.381448112603\n",
      "Other\ts-0\t0.180071690849\n",
      "Other\tc-1\t0.198733003294\n",
      "Other\ts-1\t0.170800426008\n",
      "Other\tc-0\t0.307145557827\n",
      "American\tneither\t0.453949528272\n",
      "American\tc-0\t0.265978473064\n",
      "American\tc-1\t0.153505336926\n",
      "American\ts-1\t0.144071347516\n",
      "American\ts-0\t0.139064663057\n"
     ]
    }
   ],
   "source": [
    "# calc avg % strategies (fine-grain)\n",
    "# 2 categories: US & Other\n",
    "# import statistics as stat\n",
    "other_spkr_ratios = defaultdict(list)\n",
    "for nat, spkrs in nat_hifreq.iteritems():\n",
    "    if nat == 'US':\n",
    "        continue\n",
    "    \n",
    "    for spkr in spkrs:\n",
    "        if spkr not in strat_ratio:\n",
    "            continue\n",
    "        for strat, val in strat_ratio[spkr].iteritems():\n",
    "            other_spkr_ratios[strat].append(val)\n",
    "            \n",
    "us_spkr_ratios = defaultdict(list)\n",
    "for spkr in nat_hifreq['US']:\n",
    "    if spkr not in strat_ratio:\n",
    "        continue\n",
    "    for strat, val in strat_ratio[spkr].iteritems():\n",
    "        us_spkr_ratios[strat].append(val)\n",
    "            \n",
    "for strat_tot, strat_lst in other_spkr_ratios.iteritems():\n",
    "    print '{}\\t{}\\t{}'.format('Other', strat_tot, stat.mean(strat_lst))\n",
    "for strat_tot, strat_lst in us_spkr_ratios.iteritems():\n",
    "    print '{}\\t{}\\t{}'.format('American', strat_tot, stat.mean(strat_lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Other\ts\t0.26605952336\n",
      "Other\tc\t0.380747779785\n",
      "Other\tn\t0.353192696854\n",
      "American\ts\t0.225335614875\n",
      "American\tc\t0.341348926319\n",
      "American\tn\t0.433315458805\n"
     ]
    }
   ],
   "source": [
    "# calc avg % strategies (coarse-grain)\n",
    "# 2 categories: US & Other\n",
    "# import statistics as stat\n",
    "other_spkr_ratios = defaultdict(list)\n",
    "for nat, spkrs in nat_hifreq.iteritems():\n",
    "    if nat == 'US':\n",
    "        continue\n",
    "    \n",
    "    for spkr in spkrs:\n",
    "        if spkr not in spkr_csid:\n",
    "            continue\n",
    "        for strat, val in coarse_strat_ratio[spkr].iteritems():\n",
    "            other_spkr_ratios[strat].append(val)\n",
    "            \n",
    "us_spkr_ratios = defaultdict(list)\n",
    "for spkr in nat_hifreq['US']:\n",
    "    if spkr not in spkr_csid:\n",
    "        continue\n",
    "    for strat, val in coarse_strat_ratio[spkr].iteritems():\n",
    "        us_spkr_ratios[strat].append(val)\n",
    "            \n",
    "for strat_tot, strat_lst in other_spkr_ratios.iteritems():\n",
    "    print '{}\\t{}\\t{}'.format('Other', strat_tot, stat.mean(strat_lst))\n",
    "for strat_tot, strat_lst in us_spkr_ratios.iteritems():\n",
    "    print '{}\\t{}\\t{}'.format('American', strat_tot, stat.mean(strat_lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PR\ts\t0.188522825151\n",
      "PR\tc\t0.508161068045\n",
      "PR\tn\t0.303316106804\n",
      "Dominican\ts\t0.221154949146\n",
      "Dominican\tc\t0.449915344373\n",
      "Dominican\tn\t0.32892970648\n",
      "Cuba\ts\t0.253267863585\n",
      "Cuba\tc\t0.387893241524\n",
      "Cuba\tn\t0.358838894891\n",
      "Venezuela\ts\t0.245213113713\n",
      "Venezuela\tc\t0.396704393433\n",
      "Venezuela\tn\t0.358082492854\n",
      "US\ts\t0.236288522398\n",
      "US\tc\t0.371850918402\n",
      "US\tn\t0.3918605592\n",
      "5\ts\t0.242391934787\n",
      "5\tc\t0.367493944181\n",
      "5\tn\t0.390114121032\n",
      "Colombia\ts\t0.253998413978\n",
      "Colombia\tc\t0.35697272357\n",
      "Colombia\tn\t0.389028862452\n",
      "US/Cuba\ts\t0.257919654604\n",
      "US/Cuba\tc\t0.363265007395\n",
      "US/Cuba\tn\t0.378815338001\n",
      "Nicaragua\ts\t0.251576487927\n",
      "Nicaragua\tc\t0.367883309434\n",
      "Nicaragua\tn\t0.380540202639\n",
      "Ecuador\ts\t0.254271023536\n",
      "Ecuador\tc\t0.369342848519\n",
      "Ecuador\tn\t0.376386127945\n"
     ]
    }
   ],
   "source": [
    "# calc avg % strategies (coarse-grain)\n",
    "# all non-US categories\n",
    "other_spkr_ratios = defaultdict(list)\n",
    "for nat, spkrs in nat_hifreq.iteritems():\n",
    "#     if nat == 'US':\n",
    "#         continue\n",
    "    \n",
    "    for spkr in spkrs:\n",
    "        if spkr not in spkr_csid:\n",
    "            continue\n",
    "        for strat, val in coarse_strat_ratio[spkr].iteritems():\n",
    "            other_spkr_ratios[strat].append(val)\n",
    "    for strat_tot in ['s', 'c', 'n']:\n",
    "        print '{}\\t{}\\t{}'.format(nat, strat_tot, stat.mean(other_spkr_ratios[strat_tot]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# get distribution of num utts by nationality\n",
    "# print spkr_numutts\n",
    "for nat, spkrs in nat_hifreq.iteritems():\n",
    "    if nat != 'US':\n",
    "        continue\n",
    "#     print spkrs\n",
    "    for spkr in spkrs:\n",
    "        if spkr not in spkr_csid:\n",
    "            continue\n",
    "        print spkr_numutts[spkr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dom_spa\tneither\t0.398085016835\n",
      "dom_spa\ts-0\t0.128914141414\n",
      "dom_spa\tc-1\t0.22037037037\n",
      "dom_spa\ts-1\t0.144848484848\n",
      "dom_spa\tc-0\t0.41632996633\n",
      "sym\tneither\t0.401319497691\n",
      "sym\tc-0\t0.308176572059\n",
      "sym\tc-1\t0.187952815418\n",
      "sym\ts-1\t0.165585206443\n",
      "sym\ts-0\t0.172304852518\n",
      "dom_eng\tneither\t0.412780933913\n",
      "dom_eng\ts-0\t0.163443484371\n",
      "dom_eng\ts-1\t0.160642588824\n",
      "dom_eng\tc-1\t0.167157367973\n",
      "dom_eng\tc-0\t0.147859446779\n"
     ]
    }
   ],
   "source": [
    "# calc lang strat (fine)\n",
    "lang_spkr_ratios = {}\n",
    "for lang_prof, spkrs in lang_map.iteritems():\n",
    "    lang_spkr_ratios[lang_prof] = defaultdict(list)\n",
    "    for spkr in spkrs:\n",
    "        if spkr not in strat_ratio:\n",
    "            continue\n",
    "            \n",
    "        for strat, val in strat_ratio[spkr].iteritems():\n",
    "            lang_spkr_ratios[lang_prof][strat].append(val)\n",
    "\n",
    "for lang_key, lang_strats in lang_spkr_ratios.iteritems():\n",
    "    for strat_tot, strat_lst in lang_strats.iteritems():\n",
    "        print '{}\\t{}\\t{}'.format(lang_key, strat_tot, stat.mean(strat_lst))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dom_spa\ts\t0.154987373737\n",
      "dom_spa\tc\t0.446927609428\n",
      "dom_spa\tn\t0.398085016835\n",
      "sym\ts\t0.259331301593\n",
      "sym\tc\t0.375181298724\n",
      "sym\tn\t0.365487399683\n",
      "dom_eng\ts\t0.296845492466\n",
      "dom_eng\tc\t0.290373573622\n",
      "dom_eng\tn\t0.412780933913\n"
     ]
    }
   ],
   "source": [
    "# calc lang strat (fine)\n",
    "lang_spkr_ratios = {}\n",
    "for lang_prof, spkrs in lang_map.iteritems():\n",
    "    lang_spkr_ratios[lang_prof] = defaultdict(list)\n",
    "    for spkr in spkrs:\n",
    "        if spkr not in strat_ratio:\n",
    "            continue\n",
    "            \n",
    "        for strat, val in coarse_strat_ratio[spkr].iteritems():\n",
    "            lang_spkr_ratios[lang_prof][strat].append(val)\n",
    "\n",
    "for lang_key, lang_strats in lang_spkr_ratios.iteritems():\n",
    "    for strat_tot, strat_lst in lang_strats.iteritems():\n",
    "        print '{}\\t{}\\t{}'.format(lang_key, strat_tot, stat.mean(strat_lst))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# get distribution of num utts by lang prof\n",
    "# print spkr_numutts\n",
    "for lang_prof, spkrs in lang_map.iteritems():\n",
    "    print lang_prof\n",
    "    for spkr in spkrs:\n",
    "        if spkr not in strat_ratio:\n",
    "            continue\n",
    "        print spkr_numutts[spkr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}