step1.1_submit_10-fold_batch_jobs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pcntoolkit as ptk\n",
    "from itertools import product\n",
    "from functools import reduce\n",
    "from pcntoolkit.model.SHASH import SHASHb, SHASH, SHASHo\n",
    "from scipy.stats import gaussian_kde\n",
    "import pymc as pm\n",
    "import scipy.special as spp\n",
    "import arviz as av\n",
    "from scipy.stats import skew, kurtosis\n",
    "import seaborn as sns\n",
    "sns.set(style='darkgrid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "projdir = '/project_cephfs/3022017.02/projects/stijdboe/make_results/10_folds_results/'\n",
    "data_dir = '/project_cephfs/3022017.02/projects/stijdboe/Data'\n",
    "folds_dir = os.path.join(data_dir,'10_folds_sexcov')\n",
    "python_path = '/home/preclineu/stijdboe/.conda/envs/docker_venv/bin/python'\n",
    "normative_path = '/home/preclineu/stijdboe/.conda/envs/docker_venv/lib/python3.9/site-packages/pcntoolkit-0.28-py3.9.egg/pcntoolkit/normative.py'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size= 1\n",
    "n_chains= '2'\n",
    "n_cores_per_batch= '4'  # <- this specifies the number of cores for the job\n",
    "cores = '4'           # <- this specifies the number of cores for pymc3\n",
    "n_samples = '1000'\n",
    "n_tuning='500'\n",
    "memory = '30gb'\n",
    "inscaler = 'standardize'\n",
    "outscaler = 'standardize'\n",
    "method='bspline'\n",
    "linear_mu = 'True'\n",
    "random_intercept_mu='True'\n",
    "linear_sigma   = 'True'\n",
    "target_accept = '0.99'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_names = ['SHASHo','SHASHb_1','SHASHb_2','Normal']\n",
    "# model_names = ['SHASHb_1','SHASHb_2']\n",
    "\n",
    "likelihood_map = {'SHASHb_1':'SHASHb','SHASHb_2':'SHASHb','SHASHo':'SHASHo','Normal':'Normal'}\n",
    "durationmap = {'Normal':'12:00:00','SHASHb':'50:00:00','SHASHo':'25:00:00'}\n",
    "epsilon_linear_map = {'SHASHb_1':'False','SHASHb_2':'True','Normal':'False','SHASHo':'False'}\n",
    "delta_linear_map = {'SHASHb_1':'False','SHASHb_2':'True','Normal':'False','SHASHo':'False'}\n",
    "\n",
    "for model_name in model_names:\n",
    "    likelihood = likelihood_map[model_name]\n",
    "    duration = durationmap[likelihood]\n",
    "    linear_epsilon = epsilon_linear_map[model_name]\n",
    "    linear_delta = delta_linear_map[model_name]\n",
    "    \n",
    "    for i in range(10):\n",
    "        \n",
    "        this_identifier = f\"fold_{i}_{model_name}\"\n",
    "        job_name = this_identifier\n",
    "        fold_dir = os.path.join(folds_dir,f'fold_{i}')\n",
    "        processing_dir = os.path.join(projdir, this_identifier+'/')\n",
    "        if not os.path.exists(processing_dir):\n",
    "            os.mkdir(processing_dir)\n",
    "\n",
    "        log_dir = os.path.join(processing_dir, 'log')           #\n",
    "        if not os.path.isdir(log_dir):\n",
    "            os.mkdir(log_dir)\n",
    "\n",
    "        # The paths to the data\n",
    "        X_path = os.path.join(folds_dir, f'fold_{i}', 'X_train.pkl')\n",
    "        Y_path = os.path.join(folds_dir, f'fold_{i}', 'Y_train.pkl')\n",
    "        Z_path = os.path.join(folds_dir, f'fold_{i}', 'Z_train.pkl')\n",
    "\n",
    "        ptk.normative_parallel.execute_nm(processing_dir=processing_dir,\n",
    "                                          python_path=python_path,\n",
    "                                          normative_path=normative_path,\n",
    "                                          job_name = job_name,\n",
    "                                          n_cores_per_batch = n_cores_per_batch,\n",
    "                                          cores=cores,\n",
    "                                          memory=memory,\n",
    "                                          duration=duration,\n",
    "                                          batch_size= batch_size,\n",
    "                                          \n",
    "                                          savemodel='True',\n",
    "                                          outputsuffix='estimate',\n",
    "                                          log_path=log_dir,\n",
    "                                          binary=True,\n",
    "                                 \n",
    "                                          covfile_path=X_path,\n",
    "                                          respfile_path=Y_path,\n",
    "                                          trbefile=Z_path,\n",
    "        \n",
    "                                          alg='hbr',\n",
    "                                          func='fit',\n",
    "                                          inscaler=inscaler,\n",
    "                                          outscaler=outscaler, \n",
    "                                          model_type=method,\n",
    " \n",
    "                                          likelihood = likelihood,\n",
    "                                          linear_mu=linear_mu,\n",
    "                                          random_intercept_mu=random_intercept_mu,\n",
    "                                          random_slope_mu = 'False',\n",
    "                                          random_sigma='False',\n",
    "                                          random_intercept_sigma='False',\n",
    "                                          random_slope_sigma='False',\n",
    "                                          linear_sigma=linear_sigma,\n",
    "                                          linear_epsilon=linear_epsilon,\n",
    "                                          linear_delta=linear_delta,\n",
    "                                          target_accept = target_accept,\n",
    "                                          \n",
    "                                          n_samples=n_samples,\n",
    "                                          n_tuning=n_tuning,\n",
    "                                          n_chains=n_chains,\n",
    "                                          interactive=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}