Prep_ProteinGroups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "beautiful-noise",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import sys\n",
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "established-oxford",
   "metadata": {},
   "source": [
    "<h1>get_GO_annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "hungarian-triangle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_GO_annotation(df):\n",
    "    \n",
    "    path_domain = os.path.dirname(os.path.abspath('__file__'))\n",
    "    domain_df = pd.read_csv(path_domain + '\\\\20210308_10090_GO.tab', sep='\\t')\n",
    "    \n",
    "    GO_BP_new = []\n",
    "    GO_CC_new = []\n",
    "    GO_MF_new = []\n",
    "    Subloc_new = []\n",
    "    for x in range(len(df)):\n",
    "        GO_BP = []\n",
    "        GO_CC = []\n",
    "        GO_MF = []\n",
    "        Subloc = []\n",
    "                \n",
    "\n",
    "        Pathways = []\n",
    "        for i in df['Majority protein IDs'][x]:\n",
    "            if i != '' and i in list(domain_df['Entry']):\n",
    "                GO_BP.extend(str(domain_df[domain_df['Entry']==i]['Gene ontology (biological process)'].values.tolist()[0]).split(';'))\n",
    "                GO_CC.extend(str(domain_df[domain_df['Entry']==i]['Gene ontology (cellular component)'].values.tolist()[0]).split(';'))\n",
    "                GO_MF.extend(str(domain_df[domain_df['Entry']==i]['Gene ontology (molecular function)'].values.tolist()[0]).split(';'))\n",
    "                Subloc.extend(str(domain_df[domain_df['Entry']==i]['Subcellular location [CC]'].values.tolist()[0]).split(';'))\n",
    "        GO_BP_new.append(list(set(GO_BP)))\n",
    "        GO_CC_new.append(list(set(GO_CC)))\n",
    "        GO_MF_new.append(list(set(GO_MF)))\n",
    "        Subloc_new.append(list(set(Subloc)))\n",
    "\n",
    "            \n",
    "    df['GO_BP'] = GO_BP_new\n",
    "    df['GO_CC'] = GO_CC_new\n",
    "    df['GO_MF'] = GO_MF_new\n",
    "    df['Subloc'] = Subloc_new\n",
    "    \n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "understanding-small",
   "metadata": {},
   "source": [
    "<h1> map_UniProt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "established-wrapping",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "import json\n",
    "import zlib\n",
    "from xml.etree import ElementTree\n",
    "from urllib.parse import urlparse, parse_qs, urlencode\n",
    "import requests\n",
    "from requests.adapters import HTTPAdapter, Retry\n",
    "\n",
    "POLLING_INTERVAL = 3\n",
    "\n",
    "API_URL = \"https://rest.uniprot.org\"\n",
    "\n",
    "\n",
    "retries = Retry(total=5, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504])\n",
    "session = requests.Session()\n",
    "session.mount(\"https://\", HTTPAdapter(max_retries=retries))\n",
    "\n",
    "\n",
    "def submit_id_mapping(from_db, to_db, ids):\n",
    "    request = requests.post(\n",
    "        f\"{API_URL}/idmapping/run\",\n",
    "        data={\"from\": from_db, \"to\": to_db, \"ids\": \",\".join(ids)},\n",
    "    )\n",
    "    request.raise_for_status()\n",
    "    return request.json()[\"jobId\"]\n",
    "\n",
    "def get_next_link(headers):\n",
    "    re_next_link = re.compile(r'<(.+)>; rel=\"next\"')\n",
    "    if \"Link\" in headers:\n",
    "        match = re_next_link.match(headers[\"Link\"])\n",
    "        if match:\n",
    "            return match.group(1)\n",
    "\n",
    "\n",
    "def check_id_mapping_results_ready(job_id):\n",
    "    while True:\n",
    "        request = session.get(f\"{API_URL}/idmapping/status/{job_id}\")\n",
    "        request.raise_for_status()\n",
    "        j = request.json()\n",
    "        if \"jobStatus\" in j:\n",
    "            if j[\"jobStatus\"] == \"RUNNING\":\n",
    "                print(f\"Retrying in {POLLING_INTERVAL}s\")\n",
    "                time.sleep(POLLING_INTERVAL)\n",
    "            else:\n",
    "                raise Exception(request[\"jobStatus\"])\n",
    "        else:\n",
    "            return bool(j[\"results\"] or j[\"failedIds\"])\n",
    "\n",
    "\n",
    "def get_batch(batch_response, file_format, compressed):\n",
    "    batch_url = get_next_link(batch_response.headers)\n",
    "    while batch_url:\n",
    "        batch_response = session.get(batch_url)\n",
    "        batch_response.raise_for_status()\n",
    "        yield decode_results(batch_response, file_format, compressed)\n",
    "        batch_url = get_next_link(batch_response.headers)\n",
    "\n",
    "\n",
    "def combine_batches(all_results, batch_results, file_format):\n",
    "    if file_format == \"json\":\n",
    "        for key in (\"results\", \"failedIds\"):\n",
    "            if key in batch_results and batch_results[key]:\n",
    "                all_results[key] += batch_results[key]\n",
    "    elif file_format == \"tsv\":\n",
    "        return all_results + batch_results[1:]\n",
    "    else:\n",
    "        return all_results + batch_results\n",
    "    return all_results\n",
    "\n",
    "\n",
    "def get_id_mapping_results_link(job_id):\n",
    "    url = f\"{API_URL}/idmapping/details/{job_id}\"\n",
    "    request = session.get(url)\n",
    "    request.raise_for_status()\n",
    "    return request.json()[\"redirectURL\"]\n",
    "\n",
    "\n",
    "def decode_results(response, file_format, compressed):\n",
    "    if compressed:\n",
    "        decompressed = zlib.decompress(response.content, 16 + zlib.MAX_WBITS)\n",
    "        if file_format == \"json\":\n",
    "            j = json.loads(decompressed.decode(\"utf-8\"))\n",
    "            return j\n",
    "        elif file_format == \"tsv\":\n",
    "            return [line for line in decompressed.decode(\"utf-8\").split(\"\\n\") if line]\n",
    "        elif file_format == \"xlsx\":\n",
    "            return [decompressed]\n",
    "        elif file_format == \"xml\":\n",
    "            return [decompressed.decode(\"utf-8\")]\n",
    "        else:\n",
    "            return decompressed.decode(\"utf-8\")\n",
    "    elif file_format == \"json\":\n",
    "        return response.json()\n",
    "    elif file_format == \"tsv\":\n",
    "        return [line for line in response.text.split(\"\\n\") if line]\n",
    "    elif file_format == \"xlsx\":\n",
    "        return [response.content]\n",
    "    elif file_format == \"xml\":\n",
    "        return [response.text]\n",
    "    return response.text\n",
    "\n",
    "\n",
    "def get_xml_namespace(element):\n",
    "    m = re.match(r\"\\{(.*)\\}\", element.tag)\n",
    "    return m.groups()[0] if m else \"\"\n",
    "\n",
    "\n",
    "def merge_xml_results(xml_results):\n",
    "    merged_root = ElementTree.fromstring(xml_results[0])\n",
    "    for result in xml_results[1:]:\n",
    "        root = ElementTree.fromstring(result)\n",
    "        for child in root.findall(\"{http://uniprot.org/uniprot}entry\"):\n",
    "            merged_root.insert(-1, child)\n",
    "    ElementTree.register_namespace(\"\", get_xml_namespace(merged_root[0]))\n",
    "    return ElementTree.tostring(merged_root, encoding=\"utf-8\", xml_declaration=True)\n",
    "\n",
    "\n",
    "def print_progress_batches(batch_index, size, total):\n",
    "    n_fetched = min((batch_index + 1) * size, total)\n",
    "    #print(f\"Fetched: {n_fetched} / {total}\")\n",
    "\n",
    "\n",
    "def get_id_mapping_results_search(url):\n",
    "    parsed = urlparse(url)\n",
    "    query = parse_qs(parsed.query)\n",
    "    file_format = query[\"format\"][0] if \"format\" in query else \"json\"\n",
    "    if \"size\" in query:\n",
    "        size = int(query[\"size\"][0])\n",
    "    else:\n",
    "        size = 500\n",
    "        query[\"size\"] = size\n",
    "    compressed = (\n",
    "        query[\"compressed\"][0].lower() == \"true\" if \"compressed\" in query else False\n",
    "    )\n",
    "    parsed = parsed._replace(query=urlencode(query, doseq=True))\n",
    "    url = parsed.geturl()\n",
    "    request = session.get(url)\n",
    "    request.raise_for_status()\n",
    "    results = decode_results(request, file_format, compressed)\n",
    "    total = int(request.headers[\"x-total-results\"])\n",
    "    print_progress_batches(0, size, total)\n",
    "    for i, batch in enumerate(get_batch(request, file_format, compressed), 1):\n",
    "        results = combine_batches(results, batch, file_format)\n",
    "        print_progress_batches(i, size, total)\n",
    "    if file_format == \"xml\":\n",
    "        return merge_xml_results(results)\n",
    "    return results\n",
    "\n",
    "\n",
    "def get_id_mapping_results_stream(url):\n",
    "    if \"/stream/\" not in url:\n",
    "        url = url.replace(\"/results/\", \"/stream/\")\n",
    "    request = session.get(url)\n",
    "    request.raise_for_status()\n",
    "    parsed = urlparse(url)\n",
    "    query = parse_qs(parsed.query)\n",
    "    file_format = query[\"format\"][0] if \"format\" in query else \"json\"\n",
    "    compressed = (\n",
    "        query[\"compressed\"][0].lower() == \"true\" if \"compressed\" in query else False\n",
    "    )\n",
    "    return decode_results(request, file_format, compressed)\n",
    "\n",
    "\n",
    "def map_UniProtID(UniProtIDs, identifier_from, identifier_to):\n",
    "    try:\n",
    "        job_id = submit_id_mapping(\n",
    "            from_db=identifier_from, to_db=identifier_to, ids=UniProtIDs \n",
    "        )\n",
    "        if check_id_mapping_results_ready(job_id):\n",
    "            link = get_id_mapping_results_link(job_id)\n",
    "            results = get_id_mapping_results_search(link)\n",
    "            # Equivalently using the stream endpoint which is more demanding\n",
    "            # on the API and so is less stable:\n",
    "            # results = get_id_mapping_results_stream(link)\n",
    "        \n",
    "    except:\n",
    "        ensemble_flat = []\n",
    "\n",
    "        \n",
    "    id_dict = {results['results'][i]['from']:results['results'][i]['to'] for i in range(len(results['results']))}\n",
    "    return id_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "labeled-vaccine",
   "metadata": {},
   "source": [
    "<h1>get_IMPI_annotation</h1> --> file import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "organized-conflict",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_IMPI_annotation(df):\n",
    "    \n",
    "    path_IMPI = os.path.dirname(os.path.abspath('__file__'))\n",
    "   \n",
    "    path_IMPI_file = path_IMPI +'\\\\IMPI_2021_Q4pre_Mus_Musculus.csv'\n",
    "    df_IMPI = pd.read_csv(path_IMPI_file)\n",
    "    \n",
    "    \n",
    "    IMPI_new = []\n",
    "    for x in range(len(df)):\n",
    "        IMPI = []\n",
    "        for i in df['ESNG'][x]:\n",
    "            if i in list(df_IMPI['Ensembl Gene ID Mus Musculus']):\n",
    "                IMPI.append(list(df_IMPI[df_IMPI['Ensembl Gene ID Mus Musculus']==i]['IMPI Class'])[0])\n",
    "            else:\n",
    "                IMPI.append('NA')\n",
    "        IMPI_new.append(IMPI)\n",
    "        \n",
    "    df['IMPI_new']=IMPI_new\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "democratic-current",
   "metadata": {},
   "source": [
    "<h1>get_MitoCarta_annotation</h1> --> file import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "exclusive-importance",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_MitoCharta_annotation(df):\n",
    "    \n",
    "    path_MC3 = os.path.dirname(os.path.abspath('__file__'))\n",
    "    df_MC3 = pd.read_excel(path_MC3+'\\\\Mouse_MitoCarta3_0.xls', sheet_name = [0,1,2])\n",
    "    \n",
    "    SubMitoLocalization_new = []\n",
    "    Pathways_new = []\n",
    "    for x in range(len(df)):\n",
    "        SubMitoLocalization = []\n",
    "        Pathways = []\n",
    "        for i in df['Entrez_GeneID'][x]:\n",
    "            if i != '' and int(i) in list(df_MC3[1]['MouseGeneID']):\n",
    "                SubMitoLocalization.append(list(df_MC3[1][df_MC3[1]['MouseGeneID']==int(i)]['MitoCarta3.0_SubMitoLocalization'])[0])\n",
    "        SubMitoLocalization_new.append(list(set(SubMitoLocalization)))\n",
    "        \n",
    "        for i in df['Entrez_GeneID'][x]:\n",
    "            if i != '' and len(list(df_MC3[1][df_MC3[1]['MouseGeneID']==int(i)]['MitoCarta3.0_MitoPathways']))>0 and list(df_MC3[1][df_MC3[1]['MouseGeneID']==int(i)]['MitoCarta3.0_MitoPathways'])!=[0]:\n",
    "                Pathways.extend(list(df_MC3[1][df_MC3[1]['MouseGeneID']==int(i)]['MitoCarta3.0_MitoPathways'])[0].split(' | '))\n",
    "        Pathways_new.append(list(set(Pathways)))\n",
    "            \n",
    "    df['SubMitoLocalization'] = SubMitoLocalization_new\n",
    "    df['Pathways'] = Pathways_new\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wireless-moisture",
   "metadata": {},
   "source": [
    "<h1>Data_filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "historic-socket",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_(df,samples=['B_','BAT_','H_','K_','L_','S_','SKM_'],cutoff = 0.75):\n",
    "    \n",
    "    valid= []\n",
    "    #get groups \n",
    "    for s in range(len(samples)):\n",
    "        col = []\n",
    "        per_val_val = []\n",
    "        values_total = []\n",
    "        [col.append(i) for i in range(len(list(df.columns))) if df.columns[i].find('Intensity '+samples[s])==0]\n",
    "        df.iloc[:,col] = np.log2((df.iloc[:,col]).replace(0,np.nan))\n",
    "        for z in range(len(df)):\n",
    "            valid_values = 0\n",
    "            valid_values = [valid_values+1 for i in range(len(col)) if ((df.iloc[z,col[i]]>0) & (df.iloc[z,col[i]]!=np.nan))]\n",
    "            per_val_val.append(sum(valid_values)/len(col))\n",
    "            values_total.append(len(col))\n",
    "        df['Valid values_'+samples[s]] = per_val_val\n",
    "        df['Values_Tissue_'+samples[s]] = values_total\n",
    "    \n",
    "    df['TissueID'] = df[['Valid values_'+samples[0],'Valid values_'+samples[1],'Valid values_'+samples[2],'Valid values_'+samples[3],'Valid values_'+samples[4],'Valid values_'+samples[5],'Valid values_'+samples[6]]].replace(0,np.nan).count(axis=1)\n",
    "        \n",
    "    \n",
    "    df = df.reset_index(drop=True)\n",
    "    \n",
    "    row_filter = [i for i in range(len(df)) if (('Verified mitochondrial' in df['IMPI_new'][i])==True or \n",
    "                                                         #('Predicted mitochondrial' in df['IMPI_new'][i])==True or \n",
    "                                                         df['SubMitoLocalization'][i]!=[])]\n",
    "    df_filtered = df.iloc[row_filter,:].reset_index(drop=True)\n",
    "\n",
    "    #Median normalization -> Substract Median log2 value of column from each member of column\n",
    "    cols = []\n",
    "    for i in range(len(list(df.columns))):\n",
    "        if df.columns[i].find('Intensity')==0:\n",
    "            cols.append(i)\n",
    "    \n",
    "    df = df.join(pd.DataFrame((df.iloc[:,cols]-df_filtered.iloc[:,cols].median()).to_numpy(), columns = list('Norm_'+df.columns[cols])))\n",
    "    \n",
    "    #z-Score across all expanded samples\n",
    "    cols = []\n",
    "    xx = pd.DataFrame()\n",
    "    \n",
    "    for i in range(len(list(df.columns))):\n",
    "        if df.columns[i].find('Norm_')==0:\n",
    "            cols.append(i)\n",
    "            \n",
    "    for i in range(len(df)):\n",
    "        #xx= xx.append(pd.Series(stats.zscore(df.iloc[i,cols].replace('NaN',np.nan), nan_policy = 'omit')),ignore_index=True)\n",
    "        xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n",
    "    xx.columns = list('Zscore_'+df.columns[cols])\n",
    "    \n",
    "    df = df.join(xx)\n",
    "    #get median of each group  \n",
    "    group = {}\n",
    "    for s in range(len(samples)):\n",
    "        col = []\n",
    "        for i in range(len(list(df.columns))):\n",
    "            if ((df.columns[i].find('Zscore_Norm_Intensity '+samples[s])==0)):\n",
    "                col.append(i)\n",
    "            group.update({samples[s]:col})\n",
    "    for i in range(len(group)):\n",
    "        df['Median_Z-score_'+list(group.keys())[i]] = df.iloc[:,(list(group.values())[i])].median(1)\n",
    "        \n",
    "        \n",
    "    #calculate valid values    \n",
    "    col_val = []\n",
    "    for i in range(len(list(df.columns))):\n",
    "        if ((df.columns[i].find('Valid values_')==0)):\n",
    "            col_val.append(i)\n",
    "    #get max valid value value across groups -> at least XX valid values in at least 1 group    \n",
    "    for i in range(len(df)):\n",
    "        if df.iloc[i,col_val].max()>=cutoff:\n",
    "            valid.append(True)\n",
    "        else:\n",
    "            valid.append(False)\n",
    "    df['valid'] = valid\n",
    "    df = df[df['valid']==True]\n",
    "    df = df.reset_index(drop=True)\n",
    "    return (df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "legal-european",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "relative-median",
   "metadata": {},
   "source": [
    "<h1>gaussian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "spectacular-import",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gaussian(df, width=0.3, downshift=-1.8, prefix=None):\n",
    "    \"\"\"\n",
    "    Impute missing values by drawing from a normal distribution\n",
    "\n",
    "    :param df:\n",
    "    :param width: Scale factor for the imputed distribution relative to the standard deviation of measured values. Can be a single number or list of one per column.\n",
    "    :param downshift: Shift the imputed values down, in units of std. dev. Can be a single number or list of one per column\n",
    "    :param prefix: The column prefix for imputed columns\n",
    "    :return:\n",
    "    \"\"\"\n",
    "\n",
    "    df = df.copy()\n",
    "\n",
    "    imputed = df.isnull()  # Keep track of what's real\n",
    "\n",
    "    if prefix:\n",
    "        mask = np.array([l.startswith(prefix) for l in df.columns.values])\n",
    "        mycols = np.arange(0, df.shape[1])[mask]\n",
    "    else:\n",
    "        mycols = np.arange(0, df.shape[1])\n",
    "    \n",
    "    if type(width) is not list:\n",
    "        width = [width] * len(mycols)\n",
    "\n",
    "    elif len(mycols) != len(width):\n",
    "        raise ValueError(\"Length of iterable 'width' does not match # of columns\")\n",
    "\n",
    "    if type(downshift) is not list:\n",
    "        downshift = [downshift] * len(mycols)\n",
    "\n",
    "    elif len(mycols) != len(downshift):\n",
    "        raise ValueError(\"Length of iterable 'downshift' does not match # of columns\")\n",
    "\n",
    "    for i in mycols:\n",
    "        data = df.iloc[:, i]\n",
    "        mask = data.isnull().values\n",
    "        mean = data.mean(axis=0)\n",
    "        stddev = data.std(axis=0)\n",
    "\n",
    "        m = mean + downshift[i]*stddev\n",
    "        s = stddev*width[i]\n",
    "\n",
    "        # Generate a list of random numbers for filling in\n",
    "        values = np.random.normal(loc=m, scale=s, size=df.shape[0])\n",
    "    \n",
    "        # Now fill them in\n",
    "        df.iloc[mask, i] = values[mask]\n",
    "\n",
    "    return df, imputed\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alternate-personal",
   "metadata": {},
   "source": [
    "<h1>get_kinase_annotation </h1> --> file import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "private-usage",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_kinase_annotation(df):\n",
    "    \n",
    "    path_kinase = os.path.dirname(os.path.abspath('__file__'))    \n",
    "    path_kinase_file = path_kinase +'\\\\pkinfam_annotated.csv'\n",
    "    df_kinase = pd.read_csv(path_kinase_file)\n",
    "    \n",
    "    kinase_new = []\n",
    "    for x in range(len(df)):\n",
    "        kinase = []\n",
    "        for i in df['Majority protein IDs'][x]:\n",
    "            if i != '' and i in list(df_kinase['UniProtID_mouse']):\n",
    "                kinase.append(list(df_kinase[df_kinase['UniProtID_mouse']==i]['Kinase_fam'])[0])\n",
    "        kinase_new.append(list(set(kinase)))    \n",
    "   \n",
    "    df['Kinase'] = kinase_new\n",
    "\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bigger-pressure",
   "metadata": {},
   "outputs": [],
   "source": [
    "def annotate_ESNG(flat_list, identifier_from, identifier_to,df):\n",
    "\n",
    "    id_dict = map_UniProtID(flat_list,identifier_from, identifier_to)\n",
    "    \n",
    "    no_match_dict = {no_match:'' for no_match in list(set(flat_list).difference(set(list(id_dict.keys()))))}\n",
    "    id_dict.update(no_match_dict)\n",
    "    \n",
    "    xx = [[re.findall(r\"ENSMUSG\\d+\",id_dict[item]) for item in df['Majority protein IDs'][i]] for i in range(len(df))]\n",
    "    return_list = [[item[0] if item != [] else '' for item in row] for row in xx]\n",
    "\n",
    "    return return_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "organized-copyright",
   "metadata": {},
   "outputs": [],
   "source": [
    "def annotate_Entrez_String(flat_list, identifier_from, identifier_to,df):\n",
    "\n",
    "    id_dict = map_UniProtID(flat_list,identifier_from, identifier_to)\n",
    "    \n",
    "    no_match_dict = {no_match:'' for no_match in list(set(flat_list).difference(set(list(id_dict.keys()))))}\n",
    "    id_dict.update(no_match_dict)\n",
    "    \n",
    "    return_list = [[id_dict[item] for item in df['Majority protein IDs'][i]] for i in range(len(df))]\n",
    "    #return_list = [[item[0] if item != [] else '' for item in row] for row in xx]\n",
    "\n",
    "    return return_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "blessed-preview",
   "metadata": {},
   "source": [
    "<h1> filter_mito"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "scheduled-maldives",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_Mito(df):\n",
    "    #df['IMPI_new']= [df['IMPI_new'][i].split(',') for i in range(len(df))]\n",
    "    row_filter = [i for i in range(len(df)) if (('Verified mitochondrial' in df['IMPI_new'][i])==True or \n",
    "                                                         #('Predicted mitochondrial' in df['IMPI_new'][i])==True or \n",
    "                                                         df['SubMitoLocalization'][i]!=[])]\n",
    "    df_filtered = df.iloc[row_filter,:].reset_index(drop=True)\n",
    "    return(df_filtered)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dimensional-romance",
   "metadata": {},
   "source": [
    "<h1>Impute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "entitled-hopkins",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute(df, prefix = 'Norm'):\n",
    "    import numpy as np\n",
    "    from scipy import stats\n",
    "    \n",
    "    cols = []\n",
    "    for i in range(len(list(df.columns))):\n",
    "        if df.columns[i].find(prefix)==0:\n",
    "            cols.append(i)\n",
    "\n",
    "    df.iloc[:,cols], t = gaussian(pd.DataFrame(df.iloc[:,cols]))\n",
    "    \n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "valued-membership",
   "metadata": {},
   "source": [
    "<h1>Formating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "stone-inventory",
   "metadata": {},
   "outputs": [],
   "source": [
    "def formating(df):\n",
    "    [df['GO_BP'][i].remove('nan') for i in range(len(df)) if ('nan' in df['GO_BP'][i])]\n",
    "    [df['GO_CC'][i].remove('nan') for i in range(len(df)) if ('nan' in df['GO_CC'][i])]\n",
    "    [df['GO_MF'][i].remove('nan') for i in range(len(df)) if ('nan' in df['GO_MF'][i])]\n",
    "    [df['Subloc'][i].remove('nan') for i in range(len(df)) if ('nan' in df['Subloc'][i])]\n",
    "    df['GO_BP'] = [';'.join(df['GO_BP'][i]) for i in range(len(df))]\n",
    "    df['GO_CC'] = [';'.join(df['GO_CC'][i]) for i in range(len(df))]\n",
    "    df['GO_MF'] = [';'.join(df['GO_MF'][i]) for i in range(len(df))]\n",
    "    df['Subloc'] = [';'.join(df['Subloc'][i]) for i in range(len(df))]\n",
    "    df['Pathways']= [';'.join(df['Pathways'][i]) for i in range(len(df))]\n",
    "    df['Majority protein IDs'] = [';'.join(df['Majority protein IDs'][i]) for i in range(len(df))]\n",
    "    df['Entrez_GeneID']= [';'.join(df['Entrez_GeneID'][i]) for i in range(len(df))]\n",
    "    df['SubMitoLocalization']= [';'.join(df['SubMitoLocalization'][i]) for i in range(len(df))]\n",
    "    df['ESNG']= [';'.join(df['ESNG'][i]) for i in range(len(df))]\n",
    "    df['IMPI_new']= [';'.join(df['IMPI_new'][i]) for i in range(len(df))]\n",
    "    df['Kinase'] = [';'.join(df['Kinase'][i]) for i in range(len(df))]\n",
    "    df['StringID']= [';'.join(df['StringID'][i]) for i in range(len(df))]\n",
    "    df[\"Protein_Selection_ID\"] = df[\"Gene names\"].astype(str)+\"_\"+df[\"Protein IDs\"]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "impaired-tribe",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.dirname(os.path.abspath('__file__'))+'\\\\MQ_output\\\\' ## might be adjusted\n",
    "\n",
    "\n",
    "\n",
    "columns_to_import = ['Protein IDs','Majority protein IDs','Peptide counts (all)','Peptide counts (razor+unique)','Peptide counts (unique)','Protein names','Gene names','Fasta headers','Number of proteins','Peptides','Razor + unique peptides','Unique peptides','Intensity B_1','Intensity B_2','Intensity B_3','Intensity B_4','Intensity B_5','Intensity B_6','Intensity BAT_1','Intensity BAT_2','Intensity BAT_3','Intensity BAT_4','Intensity BAT_5','Intensity BAT_6','Intensity H_1','Intensity H_2','Intensity H_3','Intensity H_4','Intensity H_5','Intensity H_6','Intensity K_1','Intensity K_2','Intensity K_3','Intensity K_4','Intensity K_5','Intensity K_6','Intensity L_1','Intensity L_2','Intensity L_3','Intensity L_4','Intensity L_5','Intensity L_6','Intensity S_1','Intensity S_2','Intensity S_3','Intensity S_4','Intensity S_5','Intensity S_6','Intensity SKM_1','Intensity SKM_2','Intensity SKM_3','Intensity SKM_4','Intensity SKM_5','Intensity SKM_6','Only identified by site','Reverse','Potential contaminant','id','Peptide IDs','Peptide is razor','Mod. peptide IDs','Evidence IDs','MS/MS IDs','Best MS/MS','Oxidation (M) site IDs','Phospho (STY) site IDs','Oxidation (M) site positions','Phospho (STY) site positions','Taxonomy IDs']\n",
    "df = pd.read_table((path+ 'proteinGroups.txt'), sep = '\\t', usecols= columns_to_import, low_memory=False).sort_values(by=['Gene names']).reset_index(drop=True)\n",
    "df = df[(df['Reverse']!='+') & (df['Potential contaminant']!='+')].reset_index(drop=True)\n",
    "\n",
    "df['Majority protein IDs'] = [df['Majority protein IDs'][i].split(';') for i in range(len(df))]\n",
    "\n",
    "df = get_GO_annotation(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "general-reader",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n",
      "Retrying in 3s\n"
     ]
    }
   ],
   "source": [
    "flat_list = list(set([x for xs in df['Majority protein IDs'] for x in xs]))\n",
    "\n",
    "#ensemble_dict = map_UniProtID(flat_list,'UniProtKB_AC-ID','Ensembl')\n",
    "#no_match_dict = {no_match:'' for no_match in list(set(flat_list).difference(set(list(ensemble_dict.keys()))))}\n",
    "#ensemble_dict.update(no_match_dict)\n",
    "#df['ESNG'] = [list(set([x for xs in [re.findall(r\"ENSMUSG\\d+\",ensemble_dict[item]) for item in df['Majority protein IDs'][i] if item in ensemble_dict.keys()] for x in xs])) for i in range(len(df))]\n",
    "#\n",
    "#entrez_dict = map_UniProtID(flat_list,'UniProtKB_AC-ID','GeneID')\n",
    "#no_match_dict = {no_match:'' for no_match in list(set(flat_list).difference(set(list(entrez_dict.keys()))))}\n",
    "#entrez_dict.update(no_match_dict)\n",
    "#df['Entrez_GeneID'] = [list(set([entrez_dict[item] for item in df['Majority protein IDs'][i] if item in entrez_dict.keys()])) for i in range(len(df))]\n",
    "\n",
    "df['ESNG'] = annotate_ESNG(flat_list,'UniProtKB_AC-ID','Ensembl',df)\n",
    "df['Entrez_GeneID'] = annotate_Entrez_String(flat_list,'UniProtKB_AC-ID','GeneID',df)\n",
    "#df['StringID'] = annotate(flat_list,'UniProtKB_AC-ID','STRING',df)\n",
    "\n",
    "df = get_IMPI_annotation(df)\n",
    "df = get_MitoCharta_annotation(df)\n",
    "df = get_kinase_annotation(df)\n",
    "\n",
    "df['StringID'] = annotate_Entrez_String(flat_list,'UniProtKB_AC-ID','STRING',df)\n",
    "#string_dict = map_UniProtID(flat_list,'UniProtKB_AC-ID','STRING')\n",
    "#no_match_dict = {no_match:'' for no_match in list(set(flat_list).difference(set(list(string_dict.keys()))))}\n",
    "#entrez_dict.update(string_dict)\n",
    "#df['StringID'] = [list(set([string_dict[item] for item in df['Majority protein IDs'][i] if item in string_dict.keys()])) for i in range(len(df))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "metallic-electric",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fhansen\\AppData\\Local\\Temp\\ipykernel_22324\\951938032.py:47: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n",
      "C:\\Users\\fhansen\\AppData\\Local\\Temp\\ipykernel_22324\\951938032.py:47: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n",
      "C:\\Users\\fhansen\\AppData\\Local\\Temp\\ipykernel_22324\\951938032.py:47: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n",
      "C:\\Users\\fhansen\\AppData\\Local\\Temp\\ipykernel_22324\\951938032.py:47: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n",
      "C:\\Users\\fhansen\\AppData\\Local\\Temp\\ipykernel_22324\\951938032.py:47: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n",
      "C:\\Users\\fhansen\\AppData\\Local\\Temp\\ipykernel_22324\\951938032.py:47: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  xx= xx.append(pd.Series(stats.zscore(list(df.iloc[i,cols]), nan_policy = 'omit')),ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "df_cut0_notimuted = df.copy(deep= True)\n",
    "df_cut0_notimuted = filter_(df_cut0_notimuted, cutoff = 0)\n",
    "#df_cut0_notimuted = formating(df_cut0_notimuted)\n",
    "\n",
    "df_cut05_notimuted = df.copy(deep= True)\n",
    "df_cut05_notimuted = filter_(df_cut05_notimuted, cutoff = 0.5)\n",
    "df_cut05_notimuted = formating(df_cut05_notimuted)\n",
    "\n",
    "df_cut0_notimuted_mito = df_cut0_notimuted.copy(deep= True)\n",
    "df_cut0_notimuted_mito = filter_Mito(df_cut0_notimuted_mito)\n",
    "df_cut0_notimuted_mito = formating(df_cut0_notimuted_mito)\n",
    "\n",
    "df_cut0_notimuted = formating(df_cut0_notimuted)\n",
    "\n",
    "df = filter_(df,cutoff=0.5)\n",
    "\n",
    "df_mito = df.copy(deep= True)\n",
    "df_mito = filter_Mito(df_mito)\n",
    "df_mito = formating(df_mito)\n",
    "#\n",
    "#\n",
    "df_imputed = df.copy(deep= True)\n",
    "df_imputed = impute(df_imputed)\n",
    "df_imputed = formating(df_imputed)\n",
    "\n",
    "#path_saving = os.path.dirname(os.path.abspath('__file__'))+'\\\\Prepared_tables\\\\'\n",
    "#df_mito.to_csv((path_saving+ 'Prep_Protein Groups_mito.csv'), index= False)\n",
    "#df_imputed.to_csv((path_saving+ 'Prep_Protein Groups_imputed.csv'), index= False)\n",
    "#df_cut0_notimuted.to_csv((path_saving+ 'Prep_Protein Groups_cut0_notimuted.csv'), index= False)\n",
    "#df_cut05_notimuted.to_csv((path_saving+ 'Prep_Protein Groups_cut05_notimuted.csv'), index= False)\n",
    "#df_cut0_notimuted_mito.to_csv((path_saving+ 'Prep_Protein Groups_cut0_notimuted_mito.csv'), index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "central-casting",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_saving = os.path.dirname(os.path.abspath('__file__'))+'\\\\Prepared_tables\\\\'\n",
    "df_mito.to_csv((path_saving+ 'Prep_Protein Groups_mito.csv'), index= False)\n",
    "df_imputed.to_csv((path_saving+ 'Prep_Protein Groups_imputed.csv'), index= False)\n",
    "df_cut0_notimuted.to_csv((path_saving+ 'Prep_Protein Groups_cut0_notimuted.csv'), index= False)\n",
    "df_cut05_notimuted.to_csv((path_saving+ 'Prep_Protein Groups_cut05_notimuted.csv'), index= False)\n",
    "df_cut0_notimuted_mito.to_csv((path_saving+ 'Prep_Protein Groups_cut0_notimuted_mito.csv'), index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecological-termination",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python_3.8 (Mito_eLife)",
   "language": "python",
   "name": "mito_elife"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}