single_reaction_class_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Datasets taken from https://github.com/rxn4chemistry/rxn_yields/tree/master/data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit.Chem import rdChemReactions, Draw\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import AllChem\n",
    "from descriptastorus.descriptors import rdDescriptors, rdNormalizedDescriptors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_rdkit2d_features(sm):\n",
    "    generator = rdNormalizedDescriptors.RDKit2DNormalized()\n",
    "    try:\n",
    "        features = generator.process(sm)[1:]\n",
    "    except TypeError as e:\n",
    "        return [0.] * 200\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from indigo import *\n",
    "indigo = Indigo()\n",
    "from rdkit.Chem import Draw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "was_correctly_mapped = []\n",
    "def map_r(row):\n",
    "    try:\n",
    "        r = indigo.loadReaction(row)\n",
    "        was_correctly_mapped.append(r.automap(\"discard\"))\n",
    "    except IndigoException as e:\n",
    "        print(e)\n",
    "        print(row)\n",
    "        was_correctly_mapped.append(0)\n",
    "        return None\n",
    "    return r.smiles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_num_unmapped_atoms(r):\n",
    "    if not r:\n",
    "        return [0,0]\n",
    "    num_unmapped = []\n",
    "    for s in r.split('>>'):\n",
    "        m = Chem.MolFromSmiles(s)\n",
    "        i = 0\n",
    "        for a in m.GetAtoms():\n",
    "            if a.GetAtomMapNum() == 0:\n",
    "                i += 1\n",
    "        num_unmapped.append(i)\n",
    "    return num_unmapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_reaction(r, fn):\n",
    "    rxn = AllChem.ReactionFromSmarts(r)\n",
    "    d2d = Draw.MolDraw2DCairo(1080,500)\n",
    "    d2d.DrawReaction(rxn)\n",
    "    png = d2d.GetDrawingText()\n",
    "    open(fn,'wb+').write(png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "def complete_atom_mapping(r):\n",
    "    \"\"\"\n",
    "    Add atoms to the reaction SMILES which aren't in the main product but are present in the reactants or vice versa \n",
    "    and asigns indices to them. A workaround which enables to use difference D-MPNN with incomplete reaction SMILES.\n",
    "    \"\"\"\n",
    "    global count\n",
    "    if pd.isnull(r):\n",
    "        return None\n",
    "    str_r, str_p = r.split('>>')\n",
    "    mr = Chem.MolFromSmiles(str_r)\n",
    "    mp = Chem.MolFromSmiles(str_p)\n",
    "    \n",
    "    for m in [mr, mp]:\n",
    "        unique_indices = [0]\n",
    "        for a in m.GetAtoms():\n",
    "            if a.GetAtomMapNum() in unique_indices:\n",
    "                a.SetAtomMapNum(0)\n",
    "            else:\n",
    "                unique_indices.append(a.GetAtomMapNum())\n",
    "\n",
    "    unmapped_symbols_r = [a  for a in mr.GetAtoms() if a.GetAtomMapNum() == 0]\n",
    "    unmapped_symbols_p = [a  for a in mp.GetAtoms() if a.GetAtomMapNum() == 0]\n",
    "    #maybe add some kind of substucture mapping\n",
    "    str_to_add_r, str_to_add_p = '', ''\n",
    "    max_indx = max(max([a.GetAtomMapNum() for a in mr.GetAtoms()]), max([a.GetAtomMapNum() for a in mp.GetAtoms()]))\n",
    "    for a_r in unmapped_symbols_r:\n",
    "        was_mapped = False\n",
    "        s_r = a_r.GetSymbol()\n",
    "        a_r.SetAtomMapNum(max_indx + 1)\n",
    "        for i, a_p in enumerate(unmapped_symbols_p):\n",
    "            if s_r == a_p.GetSymbol():\n",
    "                a_p.SetAtomMapNum(max_indx + 1)\n",
    "                unmapped_symbols_p.pop(i)\n",
    "                was_mapped = True\n",
    "                break\n",
    "        \n",
    "        if not was_mapped:\n",
    "            str_to_add_p += f'.[{a_r.GetSymbol()}:{max_indx + 1}]'\n",
    "        max_indx += 1\n",
    "    \n",
    "    for a_p in unmapped_symbols_p:\n",
    "        a_p.SetAtomMapNum(max_indx + 1)\n",
    "        str_to_add_r += f'.[{a_p.GetSymbol()}:{max_indx + 1}]'\n",
    "        max_indx += 1\n",
    "    \n",
    "    str_r = Chem.MolToSmiles(mr) + str_to_add_r\n",
    "    str_p = Chem.MolToSmiles(mp) + str_to_add_p\n",
    "    \n",
    "    if (not Chem.MolFromSmiles(str_r)) or (not Chem.MolFromSmiles(str_p)):\n",
    "        return None\n",
    "    \n",
    "    if Chem.MolFromSmiles(str_r).GetNumAtoms() !=  Chem.MolFromSmiles(str_p).GetNumAtoms():\n",
    "        count += 1\n",
    "#         print('>>'.join([str_r, str_p]))\n",
    "#         print(Chem.MolFromSmiles(str_r).GetNumAtoms(), Chem.MolFromSmiles(str_p).GetNumAtoms())\n",
    "#         print()\n",
    "#         return None\n",
    "\n",
    "    return '>>'.join([str_r, str_p])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Buchwald-Hartwig reactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def canonicalize_with_dict(smi, can_smi_dict={}):\n",
    "    if smi not in can_smi_dict.keys():\n",
    "        return Chem.MolToSmiles(Chem.MolFromSmiles(smi))\n",
    "    else:\n",
    "        return can_smi_dict[smi]\n",
    "\n",
    "def generate_buchwald_hartwig_rxns(df):\n",
    "    \"\"\"\n",
    "    Converts the entries in the excel files from Sandfort et al. to reaction SMILES.\n",
    "    \"\"\"\n",
    "    l  = ['IC1=NC=CC=C1', 'BrC1=NC=CC=C1', 'ClC1=NC=CC=C1']\n",
    "    rdkit_rxns = []\n",
    "    df = df.copy()\n",
    "    fwd_template = '[F,Cl,Br,I]-[c;H0;D3;+0:1](:[c,n:2]):[c,n:3].[NH2;D1;+0:4]-[c:5]>>[c,n:2]:[c;H0;D3;+0:1](:[c,n:3])-[NH;D2;+0:4]-[c:5]'\n",
    "    methylaniline = 'Cc1ccc(N)cc1'\n",
    "    pd_catalyst = Chem.MolToSmiles(Chem.MolFromSmiles('O=S(=O)(O[Pd]1~[NH2]C2C=CC=CC=2C2C=CC=CC1=2)C(F)(F)F'))\n",
    "    methylaniline_mol = Chem.MolFromSmiles(methylaniline)\n",
    "    rxn = rdChemReactions.ReactionFromSmarts(fwd_template)\n",
    "    products = []\n",
    "    for i, row in df.iterrows():\n",
    "        reacts = (Chem.MolFromSmiles(row['Aryl halide']), methylaniline_mol)\n",
    "        rxn_products = rxn.RunReactants(reacts)\n",
    "\n",
    "        rxn_products_smiles = set([Chem.MolToSmiles(mol[0]) for mol in rxn_products])\n",
    "        assert len(rxn_products_smiles) == 1\n",
    "        p  = list(rxn_products_smiles)[0]\n",
    "        if row['Aryl halide'] in l:\n",
    "            p = p.replace('c2c', 'c2n')\n",
    "            \n",
    "        products.append(p)\n",
    "    df['product'] = products\n",
    "    rxns = []\n",
    "    can_smiles_dict = {}\n",
    "    for i, row in df.iterrows():\n",
    "        aryl_halide = canonicalize_with_dict(row['Aryl halide'], can_smiles_dict)\n",
    "        can_smiles_dict[row['Aryl halide']] = aryl_halide\n",
    "        ligand = canonicalize_with_dict(row['Ligand'], can_smiles_dict)\n",
    "        can_smiles_dict[row['Ligand']] = ligand\n",
    "        base = canonicalize_with_dict(row['Base'], can_smiles_dict)\n",
    "        can_smiles_dict[row['Base']] = base\n",
    "        additive = canonicalize_with_dict(row['Additive'], can_smiles_dict)\n",
    "        can_smiles_dict[row['Additive']] = additive\n",
    "\n",
    "        reactants = f\"{aryl_halide}.{methylaniline}.{pd_catalyst}.{ligand}.{base}.{additive}\"\n",
    "        rxns.append(f\"{reactants}>>{row['product']}\")\n",
    "    return rxns, rdkit_rxns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_rdkit_features_bh  = {}\n",
    "df_bh_0 = pd.read_excel('data/Buchwald_Hartwig.xlsx', sheet_name=0)\n",
    "\n",
    "ligands = [v for v in df_bh_0['Ligand'].value_counts().index]\n",
    "base = [v for v in df_bh_0['Base'].value_counts().index]\n",
    "additive = [v for v in df_bh_0['Additive'].value_counts().index]\n",
    "cat = ['O=S(=O)(O[Pd]1~[NH2]C2C=CC=CC=2C2C=CC=CC1=2)C(F)(F)F']\n",
    "\n",
    "for l in [ligands, base, additive, cat]:\n",
    "    for sm in l:\n",
    "        if not sm:\n",
    "            f = [0.] * 200\n",
    "        else:\n",
    "            f = gen_rdkit2d_features(sm)\n",
    "        if not f or f == 'nan':\n",
    "            print(n)\n",
    "            continue\n",
    "        all_rdkit_features_bh[sm] = f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir data/mapped_Buchwald_Hartwig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 'mapped_Buchwald_Hartwig_clean'\n",
    "\n",
    "for indx, s in  zip([2768] * 10 + [3058, 3056, 3059,  3056], list(range(14))):\n",
    "    \n",
    "    df_b = pd.read_excel('data/Buchwald_Hartwig.xlsx', sheet_name=s)\n",
    "    df_b['REACTION'], rdkit_rxns = pd.Series(generate_buchwald_hartwig_rxns(df_b))\n",
    "    df_b['REACTION SHORT']  = df_b['REACTION'].apply(lambda x: '>>'.join(['.'.join(x.split('.')[:2]), x.split('>>')[1]]))\n",
    "    mapped_reactions = df_b['REACTION SHORT'].apply(map_r)\n",
    "    mapped_reactions_sm_all_atoms = [complete_atom_mapping(r) for r in mapped_reactions]\n",
    "    \n",
    "    df_b['MAPPED REACTION'] = pd.Series(mapped_reactions_sm_all_atoms)\n",
    "\n",
    "    df_b['MAPPED REAGENTS'] = df_b['MAPPED REACTION'].apply(lambda x: x.split('>>')[0])\n",
    "    df_b['MAPPED PRODUCTS'] = df_b['MAPPED REACTION'].apply(lambda x: x.split('>>')[1])\n",
    "    df_b['YIELD'] = df_b['Output'] / 100.\n",
    "\n",
    "    train_df_b = df_b.iloc[:indx-1] # paper has starting index 1 not 0\n",
    "    test_df_b = df_b.iloc[indx-1:] # paper has starting index 1 not 0\n",
    "    \n",
    "\n",
    "    train_df_b = train_df_b.reset_index(drop=True)\n",
    "    test_df_b = test_df_b.reset_index(drop=True)\n",
    "    \n",
    "    assert len(train_df_b) + len(test_df_b) == len(pd.concat((train_df_b, test_df_b)).drop_duplicates())\n",
    "    \n",
    "    train_df_b[['MAPPED REAGENTS', 'MAPPED PRODUCTS','YIELD']].to_csv(f'data/{n}/train_cv_{s}.csv', index=False)\n",
    "    test_df_b[['MAPPED REAGENTS', 'MAPPED PRODUCTS', 'YIELD']].to_csv(f'data/{n}/test_cv_{s}.csv', index=False)\n",
    "    \n",
    "    with open(f'data/{n}/test_cv_{s}_feat_rdkit.csv', 'w') as f:\n",
    "        f.write(\"HEADER\\n\")\n",
    "        for i, r in test_df_b.iterrows():\n",
    "            f.write(','.join([','.join(map(str, all_rdkit_features_bh[r[c]])) for c in ['Ligand', 'Base', 'Additive']]))\n",
    "            f.write(',')\n",
    "            f.write(','.join(map(str, all_rdkit_features_bh[cat[0]])))\n",
    "            f.write('\\n')\n",
    "            \n",
    "    with open(f'data/{n}/train_cv_{s}_feat_rdkit.csv', 'w') as f:\n",
    "        f.write(\"HEADER\\n\")\n",
    "        for i, r in train_df_b.iterrows():\n",
    "            f.write(','.join([','.join(map(str, all_rdkit_features_bh[r[c]])) for c in ['Ligand', 'Base', 'Additive']]))\n",
    "            f.write(',')\n",
    "            f.write(','.join(map(str, all_rdkit_features_bh[cat[0]])))\n",
    "            f.write('\\n')\n",
    "\n",
    "\n",
    "    if s == 0: # for hp tuning\n",
    "        df_b_hp = df_b.sample(frac=1, random_state=0)\n",
    "        df_b_hp = df_b_hp.reset_index(drop=True)\n",
    "        \n",
    "        train_df_bh = df_b_hp.head(round(len(df_b_hp) * 0.7))\n",
    "        test_df_bh = train_df_bh.sample(frac=0.14, random_state=0)\n",
    "        train_df_bh = train_df_bh.drop(test_df_bh.index)\n",
    "        train_df_bh = train_df_bh.reset_index(drop=True)\n",
    "        test_df_bh = test_df_bh.reset_index(drop=True)\n",
    "        \n",
    "        train_df_bh[['MAPPED REAGENTS', 'MAPPED PRODUCTS','YIELD']].to_csv(f'data/{n}/train_for_hp_tune.csv', index=False)\n",
    "        test_df_bh[['MAPPED REAGENTS', 'MAPPED PRODUCTS', 'YIELD']].to_csv(f'data/{n}/test_for_hp_tune.csv', index=False)\n",
    "        \n",
    "        with open(f'data/{n}/train_for_hp_tune_feat_rdkit.csv', 'w') as f:\n",
    "            f.write(\"HEADER\\n\")\n",
    "            for i, r in train_df_bh.iterrows():\n",
    "                f.write(','.join([','.join(map(str, all_rdkit_features_bh[r[c]])) for c in ['Ligand', 'Base', 'Additive']]))\n",
    "                f.write(',')\n",
    "                f.write(','.join(map(str, all_rdkit_features_bh[cat[0]])))\n",
    "\n",
    "                f.write('\\n')\n",
    "              \n",
    "        with open(f'data/{n}/test_for_hp_tune_feat_rdkit.csv', 'w') as f:\n",
    "            f.write(\"HEADER\\n\")\n",
    "            for i, r in test_df_bh.iterrows():\n",
    "                f.write(','.join([','.join(map(str, all_rdkit_features_bh[r[c]])) for c in ['Ligand', 'Base', 'Additive']]))\n",
    "                f.write(',')\n",
    "                f.write(','.join(map(str, all_rdkit_features_bh[cat[0]])))\n",
    "                f.write('\\n')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Suzuki-Miyaura reactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "reactant_1_smiles = {\n",
    "    '6-chloroquinoline': 'C1=C(Cl)C=CC2=NC=CC=C12.CCC1=CC(=CC=C1)CC',  #CCC1=CC(=CC=C1)CC isn't used\n",
    "    '6-Bromoquinoline': 'C1=C(Br)C=CC2=NC=CC=C12.CCC1=CC(=CC=C1)CC', \n",
    "    '6-triflatequinoline': 'C1C2C(=NC=CC=2)C=CC=1OS(C(F)(F)F)(=O)=O.CCC1=CC(=CC=C1)CC',\n",
    "    '6-Iodoquinoline': 'C1=C(I)C=CC2=NC=CC=C12.CCC1=CC(=CC=C1)CC', \n",
    "    '6-quinoline-boronic acid hydrochloride': 'C1C(B(O)O)=CC=C2N=CC=CC=12.Cl.O',  #CL and O aren't used\n",
    "    'Potassium quinoline-6-trifluoroborate': '[B-](C1=CC2=C(C=C1)N=CC=C2)(F)(F)F.[K+].O',\n",
    "    '6-Quinolineboronic acid pinacol ester': 'B1(OC(C(O1)(C)C)(C)C)C2=CC3=C(C=C2)N=CC=C3.O'\n",
    "}\n",
    "\n",
    "reactant_2_smiles = {\n",
    "    '2a, Boronic Acid': 'CC1=CC=C2C(C=NN2C3OCCCC3)=C1B(O)O', \n",
    "    '2b, Boronic Ester': 'CC1=CC=C2C(C=NN2C3OCCCC3)=C1B4OC(C)(C)C(C)(C)O4', \n",
    "    '2c, Trifluoroborate': 'CC1=CC=C2C(C=NN2C3OCCCC3)=C1[B-](F)(F)F.[K+]',\n",
    "    '2d, Bromide': 'CC1=CC=C2C(C=NN2C3OCCCC3)=C1Br' \n",
    "}\n",
    "\n",
    "catalyst_smiles = {\n",
    "    'Pd(OAc)2': 'CC(=O)O~CC(=O)O~[Pd]'\n",
    "}\n",
    "\n",
    "ligand_smiles = {\n",
    "    'P(tBu)3': 'CC(C)(C)P(C(C)(C)C)C(C)(C)C', \n",
    "    'P(Ph)3 ': 'c3c(P(c1ccccc1)c2ccccc2)cccc3', \n",
    "    'AmPhos': 'CC(C)(C)P(C1=CC=C(C=C1)N(C)C)C(C)(C)C', \n",
    "    'P(Cy)3': 'C1(CCCCC1)P(C2CCCCC2)C3CCCCC3', \n",
    "    'P(o-Tol)3': 'CC1=CC=CC=C1P(C2=CC=CC=C2C)C3=CC=CC=C3C',\n",
    "    'CataCXium A': 'CCCCP(C12CC3CC(C1)CC(C3)C2)C45CC6CC(C4)CC(C6)C5', \n",
    "    'SPhos': 'COc1cccc(c1c2ccccc2P(C3CCCCC3)C4CCCCC4)OC', \n",
    "    'dtbpf': 'CC(C)(C)P(C1=CC=C[CH]1)C(C)(C)C.CC(C)(C)P(C1=CC=C[CH]1)C(C)(C)C.[Fe]', \n",
    "    'XPhos': 'P(c2ccccc2c1c(cc(cc1C(C)C)C(C)C)C(C)C)(C3CCCCC3)C4CCCCC4', \n",
    "    'dppf': 'C1=CC=C(C=C1)P([C-]2C=CC=C2)C3=CC=CC=C3.C1=CC=C(C=C1)P([C-]2C=CC=C2)C3=CC=CC=C3.[Fe+2]', \n",
    "    'Xantphos': 'O6c1c(cccc1P(c2ccccc2)c3ccccc3)C(c7cccc(P(c4ccccc4)c5ccccc5)c67)(C)C',\n",
    "    'None': ''\n",
    "}\n",
    "\n",
    "reagent_1_smiles = {\n",
    "    'NaOH': '[OH-].[Na+]', \n",
    "    'NaHCO3': '[Na+].OC([O-])=O', \n",
    "    'CsF': '[F-].[Cs+]', \n",
    "    'K3PO4': '[K+].[K+].[K+].[O-]P([O-])([O-])=O', \n",
    "    'KOH': '[K+].[OH-]', \n",
    "    'LiOtBu': '[Li+].[O-]C(C)(C)C', \n",
    "    'Et3N': 'CCN(CC)CC', \n",
    "    'None': ''\n",
    "}\n",
    "\n",
    "solvent_1_smiles = {\n",
    "    'MeCN': 'CC#N.O', \n",
    "    'THF': 'C1CCOC1.O', \n",
    "    'DMF': 'CN(C)C=O.O', \n",
    "    'MeOH': 'CO.O', \n",
    "    'MeOH/H2O_V2 9:1': 'CO.O', \n",
    "    'THF_V2': 'C1CCOC1.O'\n",
    "}\n",
    "\n",
    "def make_reaction_smiles(row):\n",
    "    precursors = f\" {reactant_1_smiles[row['Reactant_1_Name']]}.{reactant_2_smiles[row['Reactant_2_Name']]}.{catalyst_smiles[row['Catalyst_1_Short_Hand']]}.{ligand_smiles[row['Ligand_Short_Hand']]}.{reagent_1_smiles[row['Reagent_1_Short_Hand']]}.{solvent_1_smiles[row['Solvent_1_Short_Hand']]} \"\n",
    "    product = 'C1=C(C2=C(C)C=CC3N(C4OCCCC4)N=CC2=3)C=CC2=NC=CC=C12'\n",
    "    can_precursors = Chem.MolToSmiles(Chem.MolFromSmiles(precursors.replace('...', '.').replace('..', '.').replace(' .', '').replace('. ', '').replace(' ', '')))\n",
    "    can_product = Chem.MolToSmiles(Chem.MolFromSmiles(product))\n",
    "    \n",
    "    return f\"{can_precursors}>>{can_product}\"\n",
    "\n",
    "def make_short_reaction_smiles(row):\n",
    "    precursors = f\" {reactant_1_smiles[row['Reactant_1_Name']].split('.')[0]}.{reactant_2_smiles[row['Reactant_2_Name']].split('.')[0]}\" \n",
    "    product = 'C1=C(C2=C(C)C=CC3N(C4OCCCC4)N=CC2=3)C=CC2=NC=CC=C12'\n",
    "    can_precursors = Chem.MolToSmiles(Chem.MolFromSmiles(precursors.replace('...', '.').replace('..', '.').replace(' .', '').replace('. ', '').replace(' ', '')))\n",
    "    can_product = Chem.MolToSmiles(Chem.MolFromSmiles(product))\n",
    "    \n",
    "    return f\"{can_precursors}>>{can_product}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sm = pd.read_excel('data/Suzuki-Miyaura.xlsx')\n",
    "df_sm['REACTION'] = df_sm.apply(make_reaction_smiles, axis=1)\n",
    "df_sm['REACTION SHORT'] = df_sm.apply(make_short_reaction_smiles, axis=1)\n",
    "mapped_reactions_sm = df_sm['REACTION SHORT'].apply(map_r)\n",
    "mapped_reactions_sm_all_atoms = [complete_atom_mapping(r) for r in mapped_reactions_sm]\n",
    "\n",
    "df_sm['MAPPED REACTION'] = pd.Series(mapped_reactions_sm_all_atoms)\n",
    "df_sm['MAPPED REAGENTS'] = df_sm['MAPPED REACTION'].apply(lambda x: x.split('>>')[0])\n",
    "df_sm['MAPPED PRODUCTS'] = df_sm['MAPPED REACTION'].apply(lambda x: x.split('>>')[1])\n",
    "df_sm['YIELD'] = df_sm['Product_Yield_PCT_Area_UV'] / 100."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_rdkit_features_sm  = {}\n",
    "for d in [reagent_1_smiles, solvent_1_smiles, ligand_smiles, catalyst_smiles]:\n",
    "    for n, sm in d.items():\n",
    "        if not sm:\n",
    "            f = [0.] * 200\n",
    "        else:\n",
    "            f = gen_rdkit2d_features(sm)\n",
    "        if not f:\n",
    "            print(n)\n",
    "            continue\n",
    "        all_rdkit_features_sm[n] = f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir data/mapped_Suzuki_Miyaura"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_encode = ['Ligand_Short_Hand','Reagent_1_Short_Hand','Solvent_1_Short_Hand', 'Catalyst_1_Short_Hand']\n",
    "n = 'mapped_Suzuki_Miyaura'\n",
    "\n",
    "for s in range(10):\n",
    "    df_sm_f_r = df_sm.sample(frac=1, random_state=s)\n",
    "    df_sm_f_r = df_sm_f_r.reset_index(drop=True)\n",
    "    train_df_sm_f_r = df_sm_f_r.head(round(len(df_sm_f_r) * 0.7))\n",
    "    test_df_sm_f_r = df_sm_f_r.tail(round(len(df_sm_f_r) * 0.3))\n",
    "\n",
    "    train_df_sm_f_r = train_df_sm_f_r.reset_index(drop=True)\n",
    "    test_df_sm_f_r = test_df_sm_f_r.reset_index(drop=True)\n",
    "    \n",
    "    \n",
    "    assert len(df_sm_f_r) == len(train_df_sm_f_r) + len(test_df_sm_f_r)\n",
    "    assert len(train_df_sm_f_r) + len(test_df_sm_f_r) == len(pd.concat((train_df_sm_f_r, test_df_sm_f_r)).drop_duplicates())\n",
    "    \n",
    "    train_df_sm_f_r[['MAPPED REAGENTS', 'MAPPED PRODUCTS','YIELD']].to_csv(f'data/{n}/train_cv_{s}.csv', index=False)\n",
    "    test_df_sm_f_r[['MAPPED REAGENTS', 'MAPPED PRODUCTS', 'YIELD']].to_csv(f'data/{n}/test_cv_{s}.csv', index=False)\n",
    "    \n",
    "    with open(f'data/{n}/train_f{s}_feat_rdkit_reactant.csv', 'w') as f:\n",
    "        f.write(\"HEADER\\n\")\n",
    "        for i, r in train_df_sm_f_r.iterrows():\n",
    "            f.write(','.join([','.join(map(str, all_rdkit_features_sm[r[c]])) for c in columns_to_encode]))\n",
    "            f.write('\\n')\n",
    "        \n",
    "    with open(f'data/{n}/test_f{s}_feat_rdkit_reactant.csv', 'w') as f:\n",
    "        f.write(\"HEADER\\n\")\n",
    "        for i, r in test_df_sm_f_r.iterrows():\n",
    "            f.write(','.join([','.join(map(str, all_rdkit_features_sm[r[c]])) for c in columns_to_encode]))\n",
    "            f.write('\\n')\n",
    "            \n",
    "    if s == 0: #for hp tu \n",
    "        df_sm_f_r = df_sm_f_r.sample(frac=1, random_state=0)\n",
    "        df_sm_f_r = df_sm_f_r.reset_index(drop=True)\n",
    "        train_df_sm_f_r = df_sm_f_r.head(round(len(df_sm_f_r) * 0.7))\n",
    "        test_df_sm_f_r = train_df_sm_f_r.sample(frac=0.14, random_state=0)\n",
    "        train_df_sm_f_r = train_df_sm_f_r.drop(test_df_sm_f_r.index)\n",
    "        train_df_sm_f_r = train_df_sm_f_r.reset_index(drop=True)\n",
    "        test_df_sm_f_r = test_df_sm_f_r.reset_index(drop=True)\n",
    "        train_df_sm_f_r[['MAPPED REAGENTS', 'MAPPED PRODUCTS','YIELD']].to_csv(f'data/{n}/train_for_hp_tune.csv', index=False)\n",
    "        test_df_sm_f_r[['MAPPED REAGENTS', 'MAPPED PRODUCTS', 'YIELD']].to_csv(f'data/{n}/test_for_hp_tune.csv', index=False)\n",
    "        \n",
    "        with open(f'data/{n}/train_for_hp_tune_feat_rdkit.csv', 'w') as f:\n",
    "            f.write(\"HEADER\\n\")\n",
    "            for i, r in train_df_sm_f_r.iterrows():\n",
    "                f.write(','.join([','.join(map(str, all_rdkit_features_sm[r[c]])) for c in columns_to_encode]))\n",
    "                f.write('\\n')\n",
    "        \n",
    "        with open(f'data/{n}/test_for_hp_tune_feat_rdkit.csv', 'w') as f:\n",
    "            f.write(\"HEADER\\n\")\n",
    "            for i, r in test_df_sm_f_r.iterrows():\n",
    "                f.write(','.join([','.join(map(str, all_rdkit_features_sm[r[c]])) for c in columns_to_encode]))\n",
    "                f.write('\\n')\n",
    "\n",
    "        \n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}