notebooks/add_additional_confounds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ba57d764",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_14388/2366672354.py:22: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import display, HTML\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#notebook { padding-top:0px !important; } .container { width:100% !important; } .end_space { min-height:0px !important; } </style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import shutil\n",
    "from bids import BIDSLayout\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from io import StringIO\n",
    "import re\n",
    "import nibabel as nb\n",
    "#from nilearn import image as nli\n",
    "from subprocess import run, PIPE\n",
    "from collections import namedtuple\n",
    "import json\n",
    "from matplotlib import pyplot as plt\n",
    "from joblib import Parallel, delayed, parallel_backend\n",
    "%matplotlib inline\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_colwidth', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>\"\n",
    "    + \"#notebook { padding-top:0px !important; } \" \n",
    "    + \".container { width:100% !important; } \"\n",
    "    + \".end_space { min-height:0px !important; } \"\n",
    "    + \"</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57de1e92",
   "metadata": {},
   "outputs": [],
   "source": [
    "project_root = Path('..')\n",
    "bids_dir = project_root / 'data/bids_defaced'\n",
    "derivatives_dir = project_root / 'data/derivatives'\n",
    "swarm_cmd_dir = project_root / 'swarm/fmriprep/swarm_cmds'\n",
    "# sing_img_dir = Path('/data/MBDU/singularity_images/')\n",
    "# image_path = (sing_img_dir/'fmriprep_v21.0.0.simg').as_posix()\n",
    "# fs_licence_path = sing_img_dir/'license.txt'\n",
    "# assert fs_licence_path.exists()\n",
    "# fs_licence_path = fs_licence_path.as_posix()\n",
    "\n",
    "# change this for other fmriprep runs\n",
    "#run_name = 'rn_aroma'\n",
    "#fmriprep_out = derivatives_dir / 'fmriprep' / run_name\n",
    "\n",
    "#cmd_file = swarm_cmd_dir / run_name\n",
    "#swarm_log_dir = project_root / ('swarm/fmriprep/swarm_logs' + run_name)\n",
    "jobids = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ebf9301d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change this for other fmriprep runs\n",
    "#run_name = 'fmriprepv20.2.2_complete_rerun'\n",
    "run_name = 'fmriprep_v21.0.0'\n",
    "fmriprep_out = derivatives_dir / 'fmriprep' / run_name\n",
    "toc_dir = derivatives_dir / 'fitlins' / run_name\n",
    "fmriprep_out.exists()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cf3c1534",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.4 ms, sys: 3.94 ms, total: 21.4 ms\n",
      "Wall time: 125 ms\n"
     ]
    }
   ],
   "source": [
    "database_path='/data/MBDU/mid_share/notebooks/pybids140'\n",
    "%time layout = BIDSLayout(bids_dir, database_path=database_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bf99e05b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# subs = !ls /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprepv20.2.2/\n",
    "# subs = subs[1:]\n",
    "subs = layout.get(return_type='id', target='subject', suffix='T1w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a1ad9dc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "preproc_glob = 'sub-*/out/sub-*/ses-*/func/sub-*_ses-*_task-*_run-*_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz'\n",
    "bold_glob = 'sub-*/ses-*/func/sub-*_ses-*_task-*_run-*_bold.nii.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d1592fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "preproced = sorted(fmriprep_out.glob(preproc_glob))\n",
    "ppdf = []\n",
    "for pp in preproced:\n",
    "    row = {}\n",
    "    ents = layout.parse_file_entities(pp.as_posix())\n",
    "    row.update(ents)\n",
    "    row['entities'] = ents\n",
    "    row['file'] = pp\n",
    "    ppdf.append(row)\n",
    "ppdf = pd.DataFrame(ppdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4dd4d6f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = sorted(bids_dir.glob(bold_glob))\n",
    "rwdf = []\n",
    "for pp in raw:\n",
    "    row = layout.parse_file_entities(pp.as_posix())\n",
    "    row['file'] = pp\n",
    "    if 'echo' in row.keys():\n",
    "        if row['echo'] == '1':\n",
    "            rwdf.append(row)\n",
    "    else:\n",
    "        rwdf.append(row)\n",
    "rwdf = pd.DataFrame(rwdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f07ef291",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_subs = []\n",
    "for sub in subs:\n",
    "    sn = sub.split('-')[-1]\n",
    "    if not sn in ppdf.subject.values:\n",
    "        missing_subs.append(sub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "16b84a85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['24085']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_subs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9f5d1240",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_sns = [ms.split(\"-\")[-1] for ms in missing_subs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bbad3f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "alldf = rwdf.merge(ppdf, how='outer', on=['subject', 'session', 'task', 'run', 'datatype', 'extension'],\n",
    "           suffixes=['', '_pp'], indicator=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3c5e5c9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "partial_sns = alldf.query(\"_merge != 'both' & subject not in @missing_sns\").subject.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0d2d310a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['23652', '23652', '23660', '23660', '23660', '23660', '23660',\n",
       "       '23660'], dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "613f0ced",
   "metadata": {},
   "source": [
    "# add outlier and motion censoring columns to confounds file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d39f9f0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_toc(row, n_dummy=4, fd_limit=1.0, outlier_fraction_limit=0.1):\n",
    "    \n",
    "    cfd = pd.read_csv(row.cfd_new_path, sep='\\t')\n",
    "    #cfd_bak_path = row.cfd_path.replace('regressors.tsv', 'bak.bak')\n",
    "    #cfd_bak_path = Path(str(ppdf.cfd_path[0]).replace('timeseries.tsv','bak.bak'))\n",
    "    #cfd.to_csv(cfd_bak_path, index=False, na_rep='n/a', sep='\\t')\n",
    "    if row.toc_path.exists():\n",
    "        toc = np.array([int(ll.strip()) for ll in row.toc_path.read_text().split('\\n')[:-1]])\n",
    "    else:\n",
    "        raise FileNotFoundError(row.toc_path)\n",
    "    toc_frac = toc / row.ms\n",
    "    assert len(toc) == row.nv\n",
    "\n",
    "    cfd['temporal_outlier_fraction'] = toc_frac\n",
    "    cfd['censored'] = (cfd.temporal_outlier_fraction > outlier_fraction_limit) | (cfd.framewise_displacement > fd_limit)\n",
    "    cfd.censored = cfd.censored.astype(int)\n",
    "    # censor dummy scans\n",
    "    cfd.loc[:n_dummy - 1, 'censored'] = 1\n",
    "    \n",
    "    # drop previous censor columns\n",
    "    cfd = cfd.loc[:, ~cfd.columns.str.contains('censor_')]\n",
    "    # make censor columns\n",
    "    c_cols = np.zeros((len(cfd.censored),len(cfd.censored)) ).astype(int)\n",
    "    c_cols[np.diag_indices(len(cfd.censored))] = cfd.censored.values.astype(int)\n",
    "    c_cols_names = [f'censor_{nn:03d}' for nn in range(cfd.censored.sum())]\n",
    "    c_cols = pd.DataFrame(c_cols[:, cfd.censored.astype(bool)],\n",
    "                          columns=c_cols_names)\n",
    "    cfd = pd.concat([cfd, c_cols], axis=1)\n",
    "    cfd = cfd.fillna(0)\n",
    "\n",
    "    cfd.to_csv(row.cfd_new_path, index=False, na_rep='n/a', sep='\\t')\n",
    "    \n",
    "\n",
    "def make_cens_cols(df, column):\n",
    "    df = df.copy()\n",
    "    df[column] = df[column].astype(int)\n",
    "    c_cols = np.zeros((len(df[column]),len(df[column])) ).astype(int)\n",
    "    c_cols[np.diag_indices(len(df[column]))] = df[column].values.astype(int)\n",
    "    c_cols_names = [f'{column}_{nn:03d}' for nn in range(df[column].sum())]\n",
    "    c_cols = pd.DataFrame(c_cols[:, df[column].astype(bool)],\n",
    "                          columns=c_cols_names)\n",
    "    df = pd.concat([df, c_cols], axis=1)\n",
    "    df = df.fillna(0)\n",
    "    return df\n",
    "\n",
    "def expand_censor(row, fd_limit=1.0):\n",
    "    cfd = pd.read_csv(row.cfd_new_path, sep='\\t')\n",
    "    cfd = cfd.drop(cfd.columns[cfd.columns.str.contains('fdplus1_')], axis=1, errors='ignore')\n",
    "    cfd = cfd.drop(cfd.columns[cfd.columns.str.contains('strictfd_')], axis=1, errors='ignore')\n",
    "    cfd = cfd.drop(cfd.columns[cfd.columns.str.contains('strictfdplus1_')], axis=1, errors='ignore')\n",
    "    cfd = cfd.drop(cfd.columns[cfd.columns.str.contains('censfdplus1_')], axis=1, errors='ignore')\n",
    "    cfd['prev_fd'] = np.nan\n",
    "    cfd.loc[1:, 'prev_fd'] = cfd.iloc[:-1].loc[:, 'framewise_displacement'].values\n",
    "    assert cfd.columns.str.contains('censfdplus1_').sum() == 0\n",
    "    assert cfd.columns.str.contains('fdplus1_').sum() == 0\n",
    "    assert cfd.columns.str.contains('strictfd_').sum() == 0\n",
    "    assert cfd.columns.str.contains('strictfdplus1_').sum() == 0\n",
    "    \n",
    "    cfd['censfdplus1'] = (cfd.prev_fd > fd_limit) & ~(cfd.censored == 1)\n",
    "    cfd = make_cens_cols(cfd, 'censfdplus1')\n",
    "    cfd['strictfd'] = (cfd.framewise_displacement > (fd_limit/2)) & ~(cfd.censored == 1)\n",
    "    cfd = make_cens_cols(cfd, 'strictfd')\n",
    "    cfd['strictfdplus1'] = (cfd.prev_fd > (fd_limit/2)) & ~(cfd.censored == 1) & ~(cfd.strictfd == 1)\n",
    "    cfd = make_cens_cols(cfd, 'strictfdplus1')\n",
    "    cfd = cfd.drop(['censfdplus1', 'strictfd', 'strictfdplus1'], axis=1)\n",
    "    \n",
    "    cfd.to_csv(row.cfd_new_path, index=False, na_rep='n/a', sep='\\t')\n",
    "\n",
    "def calc_censored(df, prefix):\n",
    "    ccols = df.columns[df.columns.str.contains(prefix)]\n",
    "    tmp = df.loc[:, ccols]\n",
    "    return ((tmp == 1).sum(0) == 1).sum()\n",
    "\n",
    "def get_censored_cols(df, prefixes):\n",
    "    if isinstance(prefixes, str):\n",
    "        prefixes = [prefixes]\n",
    "    ccols_ind = np.array([False]*len(cfd.columns))\n",
    "    for pf in prefixes:\n",
    "        ccols_ind = ccols_ind | df.columns.str.contains(pf)\n",
    "    ccols = df.columns[ccols_ind]\n",
    "    return ccols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f98ae895",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_path(ents, updates, pattern, derivatives_dir, build_path, check_exist=True, derivatives=True):\n",
    "    mp_ents = ents.copy()\n",
    "    if derivatives:\n",
    "        mp_dir = fmriprep_out / f'sub-{ents[\"subject\"]}/out'\n",
    "    else:\n",
    "        mp_dir = derivatives_dir \n",
    "    mp_ents.update(updates)\n",
    "    mp_file = mp_dir / build_path(mp_ents, pattern, validate=False, absolute_paths=False)\n",
    "    if check_exist and not mp_file.exists():\n",
    "        raise FileNotFoundError(mp_file.as_posix())\n",
    "    else:\n",
    "        assert mp_file.parent.exists()\n",
    "    return mp_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6fc2e12f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Can't find a brain mask for these.\n",
      "473     {'subject': '23638', 'session': 'i3', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "557    {'subject': '23660', 'session': 'i15', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "Name: entities, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# add toc_file path to ppdf\n",
    "toc_pattern = 'sub-{subject}/ses-{session}/func/sub-{subject}_ses-{session}_task-{task}_run-{run}_{suffix}.{extension}'\n",
    "toc_updates = {'suffix':'toutcount', 'extension':'txt'}\n",
    "ppdf['toc_path'] = ppdf.entities.apply(lambda x: make_path(x, toc_updates, toc_pattern, derivatives_dir, layout.build_path, check_exist=False))\n",
    "\n",
    "# add mask_file path to ppdf\n",
    "img_pattern = 'sub-{subject}/ses-{session}/func/sub-{subject}_ses-{session}_task-{task}_run-{run}_space-{space}_res-{res}_desc-{desc}_{suffix}.{extension}'\n",
    "mask_updates = {'desc':'brain', 'suffix':'mask', 'extension':'nii.gz', 'res': 2}\n",
    "ppdf['mask_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                            mask_updates,\n",
    "                                                            img_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False))\n",
    "ppdf['mask_exists'] = ppdf.mask_path.apply(lambda x: x.exists())\n",
    "\n",
    "#Drop runs without a brain mask\n",
    "print(\"Can't find a brain mask for these.\")\n",
    "print(ppdf.loc[~ppdf.mask_exists, 'entities'])\n",
    "ppdf = ppdf.loc[ppdf.mask_exists].copy()\n",
    "ppdf = ppdf.drop('mask_exists', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2fc53169",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Can't find a confounds file for these.\n",
      "308       {'subject': '23546', 'session': 'i3', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "474       {'subject': '23638', 'session': 'i3', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "475        {'subject': '23638', 'session': 'i4', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "476       {'subject': '23638', 'session': 'i4', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "477        {'subject': '23638', 'session': 'i5', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "478       {'subject': '23638', 'session': 'i5', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "479        {'subject': '23638', 'session': 'i6', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "480       {'subject': '23638', 'session': 'i6', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "481        {'subject': '23638', 'session': 'i7', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "482       {'subject': '23638', 'session': 'i7', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "483        {'subject': '23638', 'session': 'i8', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "485        {'subject': '23638', 'session': 'i9', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "487        {'subject': '23638', 'session': 'v1', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "488       {'subject': '23638', 'session': 'v1', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "533      {'subject': '23652', 'session': 'i10', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "534        {'subject': '23652', 'session': 'i3', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "535       {'subject': '23652', 'session': 'i3', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "536        {'subject': '23652', 'session': 'i4', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "537       {'subject': '23652', 'session': 'i4', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "538        {'subject': '23652', 'session': 'i5', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "539       {'subject': '23652', 'session': 'i5', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "540        {'subject': '23652', 'session': 'i6', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "541       {'subject': '23652', 'session': 'i6', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "542        {'subject': '23652', 'session': 'i7', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "544        {'subject': '23652', 'session': 'v1', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "558      {'subject': '23660', 'session': 'i15', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "559       {'subject': '23660', 'session': 'i18', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "560      {'subject': '23660', 'session': 'i18', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "561        {'subject': '23660', 'session': 'i6', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "562       {'subject': '23660', 'session': 'i6', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "563        {'subject': '23660', 'session': 'i7', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "564       {'subject': '23660', 'session': 'i7', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "565        {'subject': '23660', 'session': 'i9', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "567      {'subject': '23660', 'session': 'v1i5', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "570     {'subject': '23660', 'session': 'v2i19', 'task': 'mid', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "664       {'subject': '23732', 'session': 'v4', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "859       {'subject': '23823', 'session': 'i5', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "898       {'subject': '23830', 'session': 'v4', 'task': 'rest', 'run': 1, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "1099       {'subject': '23988', 'session': 'v1', 'task': 'mid', 'run': 2, 'space': 'MNI152NLin2009cAsym', 'suffix': 'bold', 'datatype': 'func', 'extension': '.nii.gz'}\n",
      "Name: entities, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# add tsnr path\n",
    "tsnr_updates = {'desc':'tsnr', 'suffix':'bold', 'extension':'nii.gz', 'res':2}\n",
    "ppdf['tsnr_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                            tsnr_updates,\n",
    "                                                            img_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False))\n",
    "# add events path\n",
    "event_pattern = 'sub-{subject}/ses-{session}/func/sub-{subject}_ses-{session}_task-{task}_run-{run}_{suffix}.{extension}'\n",
    "event_updates = {'suffix':'events', 'extension':'tsv'}\n",
    "ppdf['events_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                            event_updates,\n",
    "                                                            event_pattern,\n",
    "                                                            bids_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                             derivatives=False))\n",
    "\n",
    "cfd_pattern = 'sub-{subject}/ses-{session}/func/sub-{subject}_ses-{session}_task-{task}_run-{run}_desc-{desc}_{suffix}.{extension}'\n",
    "# add phys1d path\n",
    "phys1d_updates = {'desc': 'physregs', 'suffix':'timeseries', 'extension':'slibase.1D'}\n",
    "ppdf['phys1d_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                              phys1d_updates,\n",
    "                                                            cfd_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                             derivatives=True))\n",
    "# cheat to get the img_meta_path\n",
    "ppdf['imgmeta_path'] = ppdf.file.apply(lambda x: Path(x.as_posix().replace(\".nii.gz\", '.json')))\n",
    "                \n",
    "# add phys path\n",
    "phys_updates = ({'suffix':'physio', 'extension':'tsv.gz'})\n",
    "ppdf['phys_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                            phys_updates,\n",
    "                                                            event_pattern,\n",
    "                                                            bids_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                            derivatives=False))\n",
    "# add phys meta path\n",
    "physmeta_updates = ({'suffix':'physio', 'extension':'json'})\n",
    "ppdf['physmeta_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                            physmeta_updates,\n",
    "                                                            event_pattern,\n",
    "                                                            bids_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                            derivatives=False))\n",
    "\n",
    "# add confounds file\n",
    "confounds_updates = {'desc': 'confounds', 'suffix':'timeseries', 'extension': 'tsv'}\n",
    "ppdf['cfd_orig_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                              confounds_updates,\n",
    "                                                            cfd_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                             derivatives=True))\n",
    "\n",
    "ppdf['cfd_orig_exists'] = ppdf.cfd_orig_path.apply(lambda x: x.exists())\n",
    "\n",
    "#Drop runs without a confounds file\n",
    "print(\"Can't find a confounds file for these.\")\n",
    "print(ppdf.loc[~ppdf.cfd_orig_exists, 'entities'])\n",
    "ppdf = ppdf.loc[ppdf.cfd_orig_exists].copy()\n",
    "ppdf = ppdf.drop('cfd_orig_exists', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "83c05875",
   "metadata": {},
   "outputs": [],
   "source": [
    "confounds_updates = {'desc': 'confounds', 'suffix':'regressors', 'extension': 'tsv'}\n",
    "ppdf['cfd_new_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                              confounds_updates,\n",
    "                                                            cfd_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                             derivatives=True))\n",
    "\n",
    "confoundsmeta_updates = {'desc': 'confounds', 'suffix':'timeseries', 'extension': 'json'}\n",
    "ppdf['cfdjson_orig_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                              confoundsmeta_updates,\n",
    "                                                            cfd_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=True,\n",
    "                                                             derivatives=True))\n",
    "\n",
    "confoundsmeta_updates = {'desc': 'confounds', 'suffix':'regressors', 'extension': 'json'}\n",
    "ppdf['cfdjson_new_path'] = ppdf.entities.apply(lambda x: make_path(x,\n",
    "                                                              confoundsmeta_updates,\n",
    "                                                            cfd_pattern,\n",
    "                                                            derivatives_dir,\n",
    "                                                            layout.build_path,\n",
    "                                                            check_exist=False,\n",
    "                                                             derivatives=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b0f69554",
   "metadata": {},
   "outputs": [],
   "source": [
    "ppdf['nv'] = ppdf.file.apply(lambda x: nb.load(x.as_posix()).get_fdata().shape[-1])\n",
    "ppdf['ms'] = ppdf.mask_path.apply(lambda x: nb.load(x.as_posix()).get_fdata().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7e721700",
   "metadata": {},
   "outputs": [],
   "source": [
    "ppdf['has_phys'] = ppdf.phys_path.apply(lambda x: x.exists())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c260cbaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 25)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_events = ~ppdf.loc[ppdf.task != 'rest', 'events_path'].apply(lambda x: x.exists())\n",
    "ppdf.loc[ppdf.task != 'rest'].loc[no_events].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "44a9d2d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(654, 25)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ppdf.loc[ppdf.task != 'rest'].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c45e53c0",
   "metadata": {},
   "source": [
    "## Create swarm to run 3dtoutcout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "aa6cd17a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_toc_cmd(row):\n",
    "    return f'module load afni; 3dToutcount -polort 2 -mask {row.mask_path} {row.file} > {row.toc_path}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "498cbd98",
   "metadata": {},
   "outputs": [],
   "source": [
    "toc_cmds = list(ppdf.apply(lambda row: make_toc_cmd(row), axis=1).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f2349ba7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-mid_run-1_toutcount.txt\n",
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-rest_run-1_toutcount.txt\n",
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-mid_run-1_toutcount.txt\n",
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-rest_run-1_toutcount.txt\n",
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-mid_run-1_toutcount.txt\n",
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_toutcount.txt\n",
      "module load afni; 3dToutcount -polort 2 -mask /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz > /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_toutcount.txt\n"
     ]
    }
   ],
   "source": [
    "run_name = 'toc'\n",
    "swarm_cmd_dir = project_root / 'swarm/fitlins/swarm_cmds'\n",
    "swarm_cmd_dir.mkdir(exist_ok=True, parents=True)\n",
    "cmd_file = swarm_cmd_dir / run_name\n",
    "swarm_log_dir = project_root / 'swarm/fitlins/swarm_logs' / run_name\n",
    "swarm_log_dir.mkdir(exist_ok=True, parents=True)\n",
    "cmd_file = Path(cmd_file.as_posix())\n",
    "cmd_file.write_text('\\n'.join(toc_cmds))\n",
    "tmp = cmd_file.read_text().split('\\n')\n",
    "for i, tt in enumerate(tmp):\n",
    "    print(tt)\n",
    "    if i > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "29ea715a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'40418225'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:100 -g 10 -t 2 --module afni --time 1:00:00 --logdir {swarm_log_dir} --job-name {run_name} --partition quick,norm\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a088a6c5",
   "metadata": {},
   "source": [
    "# Create swarm to run tsnr command"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6ced79e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_tsnr_cmd(row, n_dummy):\n",
    "    img_file_select = row.file.as_posix() + f'[{n_dummy}..$]'\n",
    "    return f'module load afni; 3dTstat -prefix {row.tsnr_path} -tsnr {img_file_select} -overwrite'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "609680a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "tsnr_cmds = list(ppdf.apply(lambda row: make_tsnr_cmd(row, 4), axis=1).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "44b60b5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n",
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v1/func/sub-20900_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n",
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n",
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-20900/out/sub-20900/ses-v4/func/sub-20900_ses-v4_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n",
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n",
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n",
      "module load afni; 3dTstat -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-tsnr_bold.nii.gz -tsnr /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz[4..$] -overwrite\n"
     ]
    }
   ],
   "source": [
    "run_name = 'tsnr'\n",
    "swarm_cmd_dir = project_root / 'swarm/fitlins/swarm_cmds'\n",
    "swarm_cmd_dir.mkdir(exist_ok=True, parents=True)\n",
    "cmd_file = swarm_cmd_dir / run_name\n",
    "swarm_log_dir = project_root / 'swarm/fitlins/swarm_logs' / run_name\n",
    "swarm_log_dir.mkdir(exist_ok=True, parents=True)\n",
    "cmd_file = Path(cmd_file.as_posix())\n",
    "cmd_file.write_text('\\n'.join(tsnr_cmds))\n",
    "tmp = cmd_file.read_text().split('\\n')\n",
    "for i, tt in enumerate(tmp):\n",
    "    print(tt)\n",
    "    if i > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f91a4fe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'40418566'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:100 -g 10 -t 2 --module afni --time 1:00:00 --logdir {swarm_log_dir} --job-name {run_name} --partition quick,norm\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b3b10c7",
   "metadata": {},
   "source": [
    "# Retrots the physio and put it in the confounds file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "c3101e78",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_retrots_command(row):\n",
    "    tr = json.loads(row.imgmeta_path.read_text())['RepetitionTime']\n",
    "    slice_list = [int((tr/2))]\n",
    "    sf = json.loads(row.physmeta_path.read_text())['SamplingFrequency']\n",
    "    cmd = f'module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file {row.phys_path} -p {sf} -n 1 -v {tr} -slice_order \"custom\" -slice_offset {slice_list} -prefix {row.phys1d_path.as_posix()[:-11]} '\n",
    "    return cmd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "c609946f",
   "metadata": {},
   "outputs": [],
   "source": [
    "retrots_cmds = list(ppdf.loc[ppdf.has_phys].apply(lambda row: make_retrots_command(row), axis=1).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "5fd75ee8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_physio.tsv.gz -p 50 -n 1 -v 2.5 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v1/func/sub-21111_ses-v1_task-rest_run-1_desc-physregs_timeseries \n",
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_physio.tsv.gz -p 50 -n 1 -v 2 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-mid_run-1_desc-physregs_timeseries \n",
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21111/ses-v5/func/sub-21111_ses-v5_task-rest_run-1_physio.tsv.gz -p 50 -n 1 -v 2.5 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21111/out/sub-21111/ses-v5/func/sub-21111_ses-v5_task-rest_run-1_desc-physregs_timeseries \n",
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21250/ses-v1/func/sub-21250_ses-v1_task-mid_run-1_physio.tsv.gz -p 50 -n 1 -v 2 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21250/out/sub-21250/ses-v1/func/sub-21250_ses-v1_task-mid_run-1_desc-physregs_timeseries \n",
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21250/ses-v1/func/sub-21250_ses-v1_task-rest_run-1_physio.tsv.gz -p 50 -n 1 -v 2.5 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21250/out/sub-21250/ses-v1/func/sub-21250_ses-v1_task-rest_run-1_desc-physregs_timeseries \n",
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21669/ses-v1/func/sub-21669_ses-v1_task-mid_run-1_physio.tsv.gz -p 50 -n 1 -v 2 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21669/out/sub-21669/ses-v1/func/sub-21669_ses-v1_task-mid_run-1_desc-physregs_timeseries \n",
      "module load afni; /gpfs/gsfs11/users/MBDU/mid_share/env/bin/python /usr/local/apps/afni/current/linux_centos_7_64/RetroTS.py  -phys_file /data/MBDU/mid_share/data/bids_defaced/sub-21669/ses-v1/func/sub-21669_ses-v1_task-rest_run-1_physio.tsv.gz -p 50 -n 1 -v 2.5 -slice_order \"custom\" -slice_offset [1] -prefix /data/MBDU/mid_share/data/derivatives/fmriprep/fmriprep_v21.0.0/sub-21669/out/sub-21669/ses-v1/func/sub-21669_ses-v1_task-rest_run-1_desc-physregs_timeseries \n"
     ]
    }
   ],
   "source": [
    "run_name = 'retrots'\n",
    "swarm_cmd_dir = project_root / 'swarm/fitlins/swarm_cmds'\n",
    "swarm_cmd_dir.mkdir(exist_ok=True, parents=True)\n",
    "cmd_file = swarm_cmd_dir / run_name\n",
    "swarm_log_dir = project_root / 'swarm/fitlins/swarm_logs' / run_name\n",
    "swarm_log_dir.mkdir(exist_ok=True, parents=True)\n",
    "cmd_file = Path(cmd_file.as_posix())\n",
    "cmd_file.write_text('\\n'.join(retrots_cmds[5:]))\n",
    "tmp = cmd_file.read_text().split('\\n')\n",
    "for i, tt in enumerate(tmp):\n",
    "    print(tt)\n",
    "    if i > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "22d6a710",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'40431117'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobids[run_name] = ! swarm -f {cmd_file} --gres=lscratch:100 -g 10 -t 2 --module afni --time 1:00:00 --logdir {swarm_log_dir} --job-name {run_name} --partition quick,norm\n",
    "jobids[run_name] = jobids[run_name][0]\n",
    "jobids[run_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "bc506ea8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_physioto_confounds(row):\n",
    "    phys_cols = [\"phys_RVT0\", \"phys_RVT1\", \"phys_RVT2\", \"phys_RVT3\", \"phys_RVT4\",\n",
    "             \"phys_Resp0\", \"phys_Resp1\", \"phys_Resp2\", \"phys_Resp3\",\n",
    "             \"phys_Card0\", \"phys_Card1\", \"phys_Card2\", \"phys_Card3\"]\n",
    "    if row.phys1d_path.exists():\n",
    "        phys_dat = pd.read_csv(row.phys1d_path, delim_whitespace=True, comment='#', names=phys_cols)\n",
    "        old_cfd = pd.read_csv(row.cfd_orig_path, sep='\\t')\n",
    "        old_cfd = old_cfd.drop(phys_cols, axis=1, errors='ignore')\n",
    "        old_cfd.iloc[0] = old_cfd.iloc[0].fillna(0)\n",
    "        if phys_cols[0] not in old_cfd.columns:\n",
    "            cfd = pd.concat([old_cfd, phys_dat], axis=1).loc[old_cfd.index]\n",
    "            # make sure none of the original confound data gets changed\n",
    "            assert (cfd.loc[:, old_cfd.columns].fillna(0) == old_cfd.fillna(0)).all().all()\n",
    "            cfd.to_csv(row.cfd_new_path, index=False, na_rep='n/a', sep='\\t')\n",
    "        else:\n",
    "            old_cfd.to_csv(row.cfd_new_path, index=False, na_rep='n/a', sep='\\t')\n",
    "    else:\n",
    "        old_cfd = pd.read_csv(row.cfd_orig_path, sep='\\t')\n",
    "        old_cfd.to_csv(row.cfd_new_path, index=False, na_rep='n/a', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "0712b535",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_new_jsons(row):\n",
    "    old_json = json.load(open(row.cfdjson_orig_path))\n",
    "    with open(row.cfdjson_new_path, \"w\") as outfile: json.dump(old_json,outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "6a10cf4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       None\n",
       "1       None\n",
       "2       None\n",
       "3       None\n",
       "4       None\n",
       "        ... \n",
       "1324    None\n",
       "1325    None\n",
       "1326    None\n",
       "1327    None\n",
       "1328    None\n",
       "Length: 1288, dtype: object"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ppdf.apply(add_physioto_confounds, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d015c768",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       None\n",
       "1       None\n",
       "2       None\n",
       "3       None\n",
       "4       None\n",
       "        ... \n",
       "1324    None\n",
       "1325    None\n",
       "1326    None\n",
       "1327    None\n",
       "1328    None\n",
       "Length: 1288, dtype: object"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ppdf.apply(add_new_jsons, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "4943e19c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add outlier and motion censoring columns to confounds file\n",
    "# run retrots\n",
    "# put retrots outputs in confounds file\n",
    "# tsnr images\n",
    "# write model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "535c85ee",
   "metadata": {},
   "source": [
    "# add censoring columns to confounds file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "68a87780",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       None\n",
       "1       None\n",
       "2       None\n",
       "3       None\n",
       "4       None\n",
       "        ... \n",
       "1324    None\n",
       "1325    None\n",
       "1326    None\n",
       "1327    None\n",
       "1328    None\n",
       "Length: 1288, dtype: object"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ppdf.apply(add_toc, axis=1)\n",
    "ppdf.apply(expand_censor, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "4baa34ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'retrots'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7beef15e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "85201f4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ppdf.to_csv(str(derivatives_dir/'summary_tables'/'prep_df.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd3d3092",
   "metadata": {},
   "source": [
    "# Save out database with prepped derivatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "bded5064",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -r '/data/MBDU/mid_share/notebooks/pybids131_with_derivs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "882e4a3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14min 1s, sys: 3min 12s, total: 17min 14s\n",
      "Wall time: 25min 8s\n"
     ]
    }
   ],
   "source": [
    "database_path='./pybids140_with_derivs'\n",
    "%time layout = BIDSLayout(bids_dir, database_path=database_path, derivatives=fmriprep_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52d2bb9b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}