1_preprocess_raw_ephys.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ff1a79bd",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0fe388cc-bff8-4adb-86d1-72b919a9fbf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spikeinterface.full as si\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from pathlib import Path\n",
    "from probeinterface.plotting import plot_probe\n",
    "import os\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c797d127",
   "metadata": {},
   "source": [
    "# set paths and choose probe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "207cce56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\ap5\\\\ap5R_2024-11-20_09-00-27_saline',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\ap5\\\\ap5R_2024-11-21_09-21-32_ap5_ctb647',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\ap5\\\\ap5R_2024-11-25_09-41-07',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\ap5\\\\ap5R_2024-11-26_09-29-29_ap5_488',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq006_2024-11-25_09-37-51',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq006_2024-11-26_09-26-05',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq006_2024-11-27_09-46-17',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq006_2024-11-28_09-48-35',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq007_2024-11-29_10-29-42',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq008_2024-11-11_14-46-19',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq008_2024-11-12_12-58-20',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq008_2024-11-13_09-11-02',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq008_2024-11-15_14-05-31',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5lr_2024-11-15_09-53-24',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5L_2024-11-18_13-59-54',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5L_2024-11-19_14-17-24',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5R_2024-11-16_12-22-37',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5R_2024-11-18_08-34-11',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5R_2024-11-19_09-28-42',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-15_09-42-15',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-16_12-18-54',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-18_13-10-16',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-19_09-52-30',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-20_09-04-54',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-21_09-25-50',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-22_09-34-41',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq007_2024-11-18_08-51-40',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq007_2024-11-19_14-14-27',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq007_2024-11-20_13-27-15']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "path_ = r\"Z:\\projects\\sequence_squad\\revision_data\\lars_recordings\\ephys\\\\\"\n",
    "\n",
    "base_recording_paths = []\n",
    "for q in os.listdir(path_):\n",
    "    if not 'other_sessions' in q:\n",
    "        folder = os.path.join(path_,q)\n",
    "        for q in os.listdir(folder):\n",
    "            base_recording_paths+=[os.path.join(folder,q)]\n",
    "            \n",
    "base_recording_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dc4bd2b",
   "metadata": {},
   "source": [
    "# make folder structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "29fc6a8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ap5R\n",
      "2024-11-20\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5R\n",
      "2024-11-21\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5R\n",
      "2024-11-25\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5R\n",
      "2024-11-26\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-25\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-26\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-27\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-28\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq007\n",
      "2024-11-29\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq008\n",
      "2024-11-11\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq008\n",
      "2024-11-12\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq008\n",
      "2024-11-13\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq008\n",
      "2024-11-15\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5lr\n",
      "2024-11-15\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5L\n",
      "2024-11-18\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5L\n",
      "2024-11-19\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5R\n",
      "2024-11-16\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5R\n",
      "2024-11-18\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "ap5R\n",
      "2024-11-19\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-15\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-16\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-18\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-19\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-20\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-21\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq006\n",
      "2024-11-22\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq007\n",
      "2024-11-18\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq007\n",
      "2024-11-19\n",
      "mouse file already exists\n",
      "recording folder already exists\n",
      "seq007\n",
      "2024-11-20\n",
      "mouse file already exists\n",
      "recording folder already exists\n"
     ]
    }
   ],
   "source": [
    "all_recording_paths = []\n",
    "for i in range(len(base_recording_paths)):\n",
    "\n",
    "    organised_path = r\"Z:\\projects\\sequence_squad\\revision_data\\organised_data\\animals\\\\\"\n",
    "\n",
    "    mouse_id = base_recording_paths[i].split('\\\\')[-1].split('_')[0]\n",
    "    print(mouse_id)\n",
    "    date_ = base_recording_paths[i].split('\\\\')[-1].split('_')[1]\n",
    "    print(date_)\n",
    "    #reverse the date\n",
    "    date_ = '-'.join(date_.split('-')[::-1])\n",
    "\n",
    "    if not os.path.isdir(organised_path+mouse_id+'_implant1'):\n",
    "        print('adding mouse file')\n",
    "        os.makedirs(organised_path+mouse_id+'_implant1')\n",
    "    else:\n",
    "        print('mouse file already exists')\n",
    "        \n",
    "    #work out what rnum will be\n",
    "    make_folder = False\n",
    "    if len(os.listdir(organised_path+mouse_id+'_implant1\\\\')) == 0:\n",
    "        r_num = '1'\n",
    "        make_folder = True\n",
    "    else:\n",
    "        r_num = str(int(os.listdir(organised_path+mouse_id+'_implant1\\\\')[-1].split('_')[0][-1]) + 1)\n",
    "        for item in os.listdir(organised_path+mouse_id+'_implant1\\\\'):\n",
    "            if not date_ in item:\n",
    "                make_folder = True\n",
    "            else:\n",
    "                make_folder = False\n",
    "                r_num = item.split('_')[0].split('ing')[-1]\n",
    "                break\n",
    "        \n",
    "    # make recording dir\n",
    "    if make_folder == True:\n",
    "        print('**making new recording folder')\n",
    "        recording_path = organised_path+mouse_id+'_implant1' + '\\\\recording' + r_num + '_' + date_ + '\\\\'\n",
    "        os.makedirs(recording_path)\n",
    "        # make folder structure \n",
    "        if not os.path.isdir(recording_path + 'ephys'):\n",
    "            os.makedirs(recording_path + 'ephys')\n",
    "        if not os.path.isdir(recording_path + 'video/tracking/'):\n",
    "            os.makedirs(recording_path + 'video/tracking/')\n",
    "        if not os.path.isdir(recording_path + 'behav_sync/'):\n",
    "            os.makedirs(recording_path + 'behav_sync/') \n",
    "            \n",
    "\n",
    "    else:\n",
    "        print('recording folder already exists')\n",
    "        recording_path = organised_path+mouse_id+'_implant1' + '\\\\recording' + r_num + '_' + date_ + '\\\\'\n",
    "    all_recording_paths+=[recording_path + 'ephys']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb1f1919",
   "metadata": {},
   "source": [
    "# chose which data to process - ie. data that hasnt yet been processed - and create save paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6d38f1b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "process_base_paths  = []\n",
    "out_paths = []\n",
    "missing_probe = []\n",
    "for index,path in enumerate(all_recording_paths):\n",
    "    if not os.path.isfile(path+ r'\\probeA\\unit_info.txt'):\n",
    "        # if no kilosort folder then add base path to process list\n",
    "        process_base_paths += [base_recording_paths[index]]\n",
    "        out_paths += [path]\n",
    "        missing_probe += ['A']\n",
    "\n",
    "    if not 'ap5' in path:\n",
    "        if not os.path.isfile(path+ r'\\probeB\\unit_info.txt'):\n",
    "            # if no kilosort folder then add base path to process list\n",
    "            process_base_paths += [base_recording_paths[index]]\n",
    "            out_paths += [path]\n",
    "            missing_probe += ['B']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dde050a-f1ce-4cc7-9b55-eceb0bca47de",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4c20c681-f035-4c80-8745-fa56e462b389",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\ap5\\\\ap5R_2024-11-21_09-21-32_ap5_ctb647',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\dual_strd_hpc\\\\seq007_2024-11-29_10-29-42',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\ap5L_2024-11-18_13-59-54',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-15_09-42-15',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-16_12-18-54',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq006_2024-11-22_09-34-41',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq007_2024-11-18_08-51-40',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq007_2024-11-19_14-14-27',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\lars_recordings\\\\ephys\\\\\\\\learning\\\\seq007_2024-11-20_13-27-15']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process_base_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fc08bc90-ebbc-4190-920d-a5c611309d9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\ap5R_implant1\\\\recording5_21-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq007_implant1\\\\recording4_29-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\ap5L_implant1\\\\recording1_18-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq006_implant1\\\\recording1_15-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq006_implant1\\\\recording2_16-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq006_implant1\\\\recording7_22-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq007_implant1\\\\recording1_18-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq007_implant1\\\\recording2_19-11-2024\\\\ephys',\n",
       " 'Z:\\\\projects\\\\sequence_squad\\\\revision_data\\\\organised_data\\\\animals\\\\\\\\seq007_implant1\\\\recording3_20-11-2024\\\\ephys']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "888a03a3",
   "metadata": {},
   "source": [
    "# kilosort loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3b06ef65-1681-43da-a1e4-cef3d4c79bd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********** PROCESSING: ***********\n",
      "Z:\\projects\\sequence_squad\\revision_data\\lars_recordings\\ephys\\\\ap5\\ap5R_2024-11-21_09-21-32_ap5_ctb647\n",
      "A\n",
      "Z:\\projects\\sequence_squad\\revision_data\\organised_data\\animals\\\\ap5R_implant1\\recording5_21-11-2024\\ephys\n",
      "['Record Node 105#NI-DAQmx-102.PXIe-6341', 'Record Node 105#Neuropix-PXI-106.ProbeA-AP', 'Record Node 105#Neuropix-PXI-106.ProbeA-LFP']\n",
      "bad_channel_ids ['AP160' 'AP165' 'AP185' 'AP216' 'AP236']\n",
      "write_binary_recording \n",
      "n_jobs=8 - samples_per_chunk=30,000 - chunk_memory=21.69 MiB - total_memory=173.49 MiB - chunk_duration=1.00s\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7434ccace3f4a79b59a76a8b139d3ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "write_binary_recording:   0%|          | 0/16971 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[13], line 62\u001b[0m\n\u001b[0;32m     60\u001b[0m \u001b[38;5;66;03m## save out the preprocessed binary\u001b[39;00m\n\u001b[0;32m     61\u001b[0m job_kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(n_jobs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m40\u001b[39m, chunk_duration\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m1s\u001b[39m\u001b[38;5;124m'\u001b[39m, progress_bar\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m---> 62\u001b[0m rec \u001b[38;5;241m=\u001b[39m \u001b[43mrec\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout_path_object\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mpreprocess\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mbinary\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mjob_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moverwrite\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m     64\u001b[0m \u001b[38;5;66;03m# here we use static plot using matplotlib backend\u001b[39;00m\n\u001b[0;32m     65\u001b[0m fig, axs \u001b[38;5;241m=\u001b[39m plt\u001b[38;5;241m.\u001b[39msubplots(ncols\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m, figsize\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m20\u001b[39m, \u001b[38;5;241m10\u001b[39m))\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\base.py:877\u001b[0m, in \u001b[0;36mBaseExtractor.save\u001b[1;34m(self, **kwargs)\u001b[0m\n\u001b[0;32m    875\u001b[0m     loaded_extractor \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msave_to_zarr(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    876\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 877\u001b[0m     loaded_extractor \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave_to_folder\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    878\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loaded_extractor\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\base.py:984\u001b[0m, in \u001b[0;36mBaseExtractor.save_to_folder\u001b[1;34m(self, name, folder, overwrite, verbose, **save_kwargs)\u001b[0m\n\u001b[0;32m    981\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msave_metadata_to_folder(folder)\n\u001b[0;32m    983\u001b[0m \u001b[38;5;66;03m# save data (done the subclass)\u001b[39;00m\n\u001b[1;32m--> 984\u001b[0m cached \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_save\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfolder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msave_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    986\u001b[0m \u001b[38;5;66;03m# copy properties/\u001b[39;00m\n\u001b[0;32m    987\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcopy_metadata(cached)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\baserecording.py:555\u001b[0m, in \u001b[0;36mBaseRecording._save\u001b[1;34m(self, format, verbose, **save_kwargs)\u001b[0m\n\u001b[0;32m    552\u001b[0m dtype \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdtype\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_dtype()\n\u001b[0;32m    553\u001b[0m t_starts \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_t_starts()\n\u001b[1;32m--> 555\u001b[0m \u001b[43mwrite_binary_recording\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfile_paths\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfile_paths\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mjob_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    557\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbinaryrecordingextractor\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BinaryRecordingExtractor\n\u001b[0;32m    559\u001b[0m \u001b[38;5;66;03m# This is created so it can be saved as json because the `BinaryFolderRecording` requires it loading\u001b[39;00m\n\u001b[0;32m    560\u001b[0m \u001b[38;5;66;03m# See the __init__ of `BinaryFolderRecording`\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\recording_tools.py:149\u001b[0m, in \u001b[0;36mwrite_binary_recording\u001b[1;34m(recording, file_paths, dtype, add_file_extension, byte_offset, auto_cast_uint, verbose, **job_kwargs)\u001b[0m\n\u001b[0;32m    145\u001b[0m init_args \u001b[38;5;241m=\u001b[39m (recording, file_path_dict, dtype, byte_offset, cast_unsigned)\n\u001b[0;32m    146\u001b[0m executor \u001b[38;5;241m=\u001b[39m ChunkRecordingExecutor(\n\u001b[0;32m    147\u001b[0m     recording, func, init_func, init_args, job_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwrite_binary_recording\u001b[39m\u001b[38;5;124m\"\u001b[39m, verbose\u001b[38;5;241m=\u001b[39mverbose, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mjob_kwargs\n\u001b[0;32m    148\u001b[0m )\n\u001b[1;32m--> 149\u001b[0m \u001b[43mexecutor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\job_tools.py:422\u001b[0m, in \u001b[0;36mChunkRecordingExecutor.run\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    419\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprogress_bar:\n\u001b[0;32m    420\u001b[0m     results \u001b[38;5;241m=\u001b[39m tqdm(results, desc\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mjob_name, total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(all_chunks))\n\u001b[1;32m--> 422\u001b[0m \u001b[43m\u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mres\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mresults\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m    423\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_returns\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m    424\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreturns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mappend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mres\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\tqdm\\notebook.py:250\u001b[0m, in \u001b[0;36mtqdm_notebook.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    248\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m    249\u001b[0m     it \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__iter__\u001b[39m()\n\u001b[1;32m--> 250\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mit\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m    251\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# return super(tqdm...) will not catch exception\u001b[39;49;00m\n\u001b[0;32m    252\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01myield\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\n\u001b[0;32m    253\u001b[0m \u001b[38;5;66;03m# NB: except ... [ as ...] breaks IPython async KeyboardInterrupt\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\site-packages\\tqdm\\std.py:1181\u001b[0m, in \u001b[0;36mtqdm.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1178\u001b[0m time \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_time\n\u001b[0;32m   1180\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1181\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43miterable\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m   1182\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01myield\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\n\u001b[0;32m   1183\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# Update and possibly print the progressbar.\u001b[39;49;00m\n\u001b[0;32m   1184\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# Note: does not call self.update(1) for speed optimisation.\u001b[39;49;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\concurrent\\futures\\process.py:620\u001b[0m, in \u001b[0;36m_chain_from_iterable_of_lists\u001b[1;34m(iterable)\u001b[0m\n\u001b[0;32m    614\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_chain_from_iterable_of_lists\u001b[39m(iterable):\n\u001b[0;32m    615\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    616\u001b[0m \u001b[38;5;124;03m    Specialized implementation of itertools.chain.from_iterable.\u001b[39;00m\n\u001b[0;32m    617\u001b[0m \u001b[38;5;124;03m    Each item in *iterable* should be a list.  This function is\u001b[39;00m\n\u001b[0;32m    618\u001b[0m \u001b[38;5;124;03m    careful not to keep references to yielded objects.\u001b[39;00m\n\u001b[0;32m    619\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 620\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43melement\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43miterable\u001b[49m\u001b[43m:\u001b[49m\n\u001b[0;32m    621\u001b[0m \u001b[43m        \u001b[49m\u001b[43melement\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreverse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    622\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mwhile\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43melement\u001b[49m\u001b[43m:\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\concurrent\\futures\\_base.py:619\u001b[0m, in \u001b[0;36mExecutor.map.<locals>.result_iterator\u001b[1;34m()\u001b[0m\n\u001b[0;32m    616\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m fs:\n\u001b[0;32m    617\u001b[0m     \u001b[38;5;66;03m# Careful not to keep a reference to the popped future\u001b[39;00m\n\u001b[0;32m    618\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 619\u001b[0m         \u001b[38;5;28;01myield\u001b[39;00m \u001b[43m_result_or_cancel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    620\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    621\u001b[0m         \u001b[38;5;28;01myield\u001b[39;00m _result_or_cancel(fs\u001b[38;5;241m.\u001b[39mpop(), end_time \u001b[38;5;241m-\u001b[39m time\u001b[38;5;241m.\u001b[39mmonotonic())\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\concurrent\\futures\\_base.py:317\u001b[0m, in \u001b[0;36m_result_or_cancel\u001b[1;34m(***failed resolving arguments***)\u001b[0m\n\u001b[0;32m    315\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m    316\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 317\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfut\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    318\u001b[0m     \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m    319\u001b[0m         fut\u001b[38;5;241m.\u001b[39mcancel()\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\concurrent\\futures\\_base.py:451\u001b[0m, in \u001b[0;36mFuture.result\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    448\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m    449\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_result()\n\u001b[1;32m--> 451\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_condition\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    453\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n\u001b[0;32m    454\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\si_env\\Lib\\threading.py:327\u001b[0m, in \u001b[0;36mCondition.wait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    325\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:    \u001b[38;5;66;03m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[39;00m\n\u001b[0;32m    326\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 327\u001b[0m         \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    328\u001b[0m         gotit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m    329\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "for ind,base_folder in enumerate(process_base_paths):\n",
    "\n",
    "    \n",
    "    # clear gpu memory cache each loop to avoid memory issues\n",
    "    torch.cuda.empty_cache()\n",
    "    print(str(ind) + 'out of' + str(len(process_base_paths)))\n",
    "    print('*********** PROCESSING: ***********')\n",
    "    print(base_folder)\n",
    "    out_path = out_paths[ind]\n",
    "    \n",
    "    print(missing_probe[ind])\n",
    "    print(out_path)\n",
    "\n",
    "\n",
    "    # extract the stream names (each np processor)\n",
    "    stream_names, stream_ids = si.get_neo_streams('openephysbinary', base_folder)\n",
    "    print(stream_names)\n",
    "\n",
    "    # chose probe id, DEAL WITH PROBE A/B stuff\n",
    "    ksort = False\n",
    "    for stream_i, stream in enumerate(stream_names):\n",
    "        if 'Probe' + missing_probe[ind] in stream:\n",
    "            if not 'LFP' in stream:\n",
    "                Probe_id = stream_names[stream_i]\n",
    "                out_path = out_path + '\\\\probe' + missing_probe[ind] + '\\\\'\n",
    "                out_path_object = Path(out_path)\n",
    "                ksort = True\n",
    "                if not os.path.isdir(out_path):\n",
    "                    os.makedirs(out_path)\n",
    "                break\n",
    "\n",
    "    if ksort == True:\n",
    "        \n",
    "        # load in data\n",
    "        raw_rec = si.read_openephys(base_folder,stream_name = Probe_id,load_sync_channel=False)\n",
    "        raw_rec.get_probe().to_dataframe()\n",
    "    \n",
    "        # plot probe\n",
    "        fig, axs = plt.subplots(figsize=(1, 100))\n",
    "        probe = raw_rec.get_probe()\n",
    "        plot_probe(probe, ax = axs)\n",
    "        plt.savefig(out_path + 'probe_map.png')\n",
    "        plt.close()\n",
    "            \n",
    "        \n",
    "        # Preprocess the recording¶\n",
    "        # Let’s do something similar to the IBL destriping chain (See :ref:ibl_destripe) to preprocess the data but:\n",
    "        # instead of interpolating bad channels, we remove then.\n",
    "        # instead of highpass_spatial_filter() we use common_reference()\n",
    "        \n",
    "        rec1 = si.highpass_filter(raw_rec, freq_min=400.)\n",
    "        bad_channel_ids, channel_labels = si.detect_bad_channels(rec1)\n",
    "        rec2 = rec1.remove_channels(bad_channel_ids)\n",
    "        print('bad_channel_ids', bad_channel_ids)\n",
    "    \n",
    "        rec3 = si.phase_shift(rec2)\n",
    "        rec4 = si.common_reference(rec3, operator=\"median\", reference=\"global\")\n",
    "        rec = rec4\n",
    "        \n",
    "        ## save out the preprocessed binary\n",
    "        job_kwargs = dict(n_jobs=40, chunk_duration='1s', progress_bar=True)\n",
    "        rec = rec.save(folder=out_path_object / 'preprocess', format='binary', **job_kwargs, overwrite = True)\n",
    "        \n",
    "        # here we use static plot using matplotlib backend\n",
    "        fig, axs = plt.subplots(ncols=3, figsize=(20, 10))\n",
    "    \n",
    "        si.plot_traces(rec1, backend='matplotlib',  clim=(-50, 50), ax=axs[0])\n",
    "        si.plot_traces(rec4, backend='matplotlib',  clim=(-50, 50), ax=axs[1])\n",
    "        si.plot_traces(rec, backend='matplotlib',  clim=(-50, 50), ax=axs[2])\n",
    "        for i, label in enumerate(('filter', 'cmr', 'final')):\n",
    "            axs[i].set_title(label)\n",
    "        plt.savefig(out_path + 'preprocessing_destriping_common_ref.png')\n",
    "        plt.close()\n",
    "        \n",
    "        \n",
    "        # plot some channels\n",
    "        fig, ax = plt.subplots(figsize=(20, 10))\n",
    "        some_chans = rec.channel_ids[[100, 150, 200, ]]\n",
    "        si.plot_traces({'filter':rec1, 'cmr': rec4}, backend='matplotlib', mode='line', ax=ax, channel_ids=some_chans)\n",
    "        plt.savefig(out_path + 'example_chans.png')\n",
    "        plt.close()\n",
    "        \n",
    "        # check noise \n",
    "        # we can estimate the noise on the scaled traces (microV) or on the raw one (which is in our case int16).\n",
    "        noise_levels_microV = si.get_noise_levels(rec, return_scaled=True)\n",
    "        noise_levels_int16 = si.get_noise_levels(rec, return_scaled=False)\n",
    "        \n",
    "        fig, ax = plt.subplots()\n",
    "        _ = ax.hist(noise_levels_microV, bins=np.arange(5, 30, 2.5))\n",
    "        ax.set_xlabel('noise  [microV]')\n",
    "        plt.savefig(out_path + 'noise_level.png')\n",
    "        plt.close()\n",
    "        \n",
    "        \n",
    "        # check default params for kilosort4\n",
    "        params_kilosort4 = si.get_default_sorter_params('kilosort4')\n",
    "        params_kilosort4['delete_recording_dat'] = False\n",
    "    \n",
    "        # # run kilosort4 with drift correction (set as True in the params)\n",
    "        sorting = si.run_sorter('kilosort4', rec, output_folder=out_path_object / 'kilosort4_output',\n",
    "                                docker_image=False, verbose=True, **params_kilosort4, remove_existing_folder = True)\n",
    "        \n",
    "        \n",
    "        ######################################################################################\n",
    "        # load back in to check quality\n",
    "        sorting = si.read_sorter_folder(out_path_object / 'kilosort4_output')\n",
    "        \n",
    "        analyzer = si.create_sorting_analyzer(sorting, rec, sparse=True, format=\"memory\")\n",
    "        \n",
    "        analyzer.compute(\"random_spikes\", method=\"uniform\", max_spikes_per_unit=500)\n",
    "        analyzer.compute(\"waveforms\",  ms_before=1.5,ms_after=2., **job_kwargs)\n",
    "        analyzer.compute(\"templates\", operators=[\"average\", \"median\", \"std\"])\n",
    "        analyzer.compute(\"noise_levels\")\n",
    "    \n",
    "        analyzer_saved = analyzer.save_as(folder=out_path_object / \"analyzer\", format=\"binary_folder\")\n",
    "    \n",
    "        metric_names=['firing_rate', 'presence_ratio', 'snr', 'isi_violation', 'amplitude_cutoff']\n",
    "    \n",
    "        metrics = si.compute_quality_metrics(analyzer, metric_names=metric_names)\n",
    "    \n",
    "        amplitude_cutoff_thresh = 0.1\n",
    "        isi_violations_ratio_thresh = 1\n",
    "        presence_ratio_thresh = 0.9\n",
    "    \n",
    "        our_query = f\"(amplitude_cutoff < {amplitude_cutoff_thresh}) & (isi_violations_ratio < {isi_violations_ratio_thresh}) & (presence_ratio > {presence_ratio_thresh})\"\n",
    "    \n",
    "        keep_units = metrics.query(our_query)\n",
    "        keep_unit_ids = keep_units.index.values\n",
    "    \n",
    "        analyzer_clean = analyzer.select_units(keep_unit_ids, folder=out_path_object / 'analyzer_clean', format='binary_folder')\n",
    "    \n",
    "        # export spike sorting report to a folder\n",
    "        si.export_report(analyzer_clean, out_path_object / 'report', format='png')\n",
    "\n",
    "        ### SAVE OUT A TXT FILE WITH NUMBER OF UNITS DATA ON IT\n",
    "        # Open a file in write mode\n",
    "        file_path = out_path + 'unit_info.txt'\n",
    "        with open(file_path, \"w\") as file:\n",
    "            # Use the file argument to save print output to the file\n",
    "            print('Kilosort output:',file = file)\n",
    "            print(sorting, file=file)\n",
    "            file.write(\"\\n\")\n",
    "            print('\"good\" units:',file = file)\n",
    "            print(analyzer_clean, file=file)\n",
    "\n",
    "        print('DONE!')\n",
    "    else:\n",
    "        print('Already processed or no probe B!')\n",
    "\n",
    "\n",
    "    \n",
    "                \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffe6614f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "28a7d329-c667-4bf7-9f73-757caae57ff1",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "'break' outside loop (3566307101.py, line 5)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  Cell \u001b[1;32mIn[12], line 5\u001b[1;36m\u001b[0m\n\u001b[1;33m    break\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m 'break' outside loop\n"
     ]
    }
   ],
   "source": [
    "# # find peaks\n",
    "\n",
    "# from spikeinterface.sortingcomponents.peak_detection import detect_peaks\n",
    "# from spikeinterface.sortingcomponents.peak_localization import localize_peaks\n",
    "# break\n",
    "# job_kwargs = dict(n_jobs=40, chunk_duration='1s', progress_bar=True)\n",
    "# peaks = detect_peaks(rec,  method='locally_exclusive', noise_levels=noise_levels_int16,\n",
    "#                      detect_threshold=5, radius_um=50., **job_kwargs)\n",
    "\n",
    "# peak_locations = localize_peaks(rec, peaks, method='center_of_mass', radius_um=50., **job_kwargs)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f0c187d-30f3-42ea-ae47-d66d3907382a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # check for drifts\n",
    "# fs = rec.sampling_frequency\n",
    "# fig, ax = plt.subplots(figsize=(10, 8))\n",
    "# ax.scatter(peaks['sample_ind'] / fs, peak_locations['y'], color='k', marker='.',  alpha=0.002)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a0080f5-0a41-426a-a31d-973fdd125d90",
   "metadata": {},
   "source": [
    "# Load back in to check quality "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66d135ac-6c14-4a35-b3df-63e402c1ec5d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7323d4b-2ee3-4dbd-b78f-28835daae20f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Emmett\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\job_tools.py:103: UserWarning: `n_jobs` is not set so parallel processing is disabled! To speed up computations, it is recommended to set n_jobs either globally (with the `spikeinterface.set_global_job_kwargs()` function) or locally (with the `n_jobs` argument). Use `spikeinterface.set_global_job_kwargs?` for more information about job_kwargs.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19e2de4f51ae406eb45316a779cfa5a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "estimate_sparsity:   0%|          | 0/14682 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SortingAnalyzer: 384 channels - 373 units - 1 segments - memory - sparse - has recording\n",
       "Loaded 0 extensions"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "analyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c020c16d-3b25-440a-8f9a-cdadc7d48a51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "407f747849b3410b950c8a36841b82dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "compute_waveforms:   0%|          | 0/14682 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SortingAnalyzer: 384 channels - 356 units - 1 segments - memory - sparse - has recording\n",
       "Loaded 4 extensions: random_spikes, waveforms, templates, noise_levels"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5d89dd9-79c0-47a3-a376-d69abe7fba61",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "79e68bfb-00e5-4eda-b10a-3c105424b383",
   "metadata": {},
   "outputs": [],
   "source": [
    "# quality metrics "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a6a63ed-5345-482b-bb8e-40c50c4fb38b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Emmett\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\qualitymetrics\\misc_metrics.py:908: UserWarning: Some units have too few spikes : amplitude_cutoff is set to NaN\n",
      "  warnings.warn(f\"Some units have too few spikes : amplitude_cutoff is set to NaN\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>firing_rate</th>\n",
       "      <th>presence_ratio</th>\n",
       "      <th>snr</th>\n",
       "      <th>isi_violations_ratio</th>\n",
       "      <th>isi_violations_count</th>\n",
       "      <th>amplitude_cutoff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.06766</td>\n",
       "      <td>0.983607</td>\n",
       "      <td>2.156246</td>\n",
       "      <td>5.724791</td>\n",
       "      <td>1078</td>\n",
       "      <td>0.00309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.276477</td>\n",
       "      <td>0.897541</td>\n",
       "      <td>1.117945</td>\n",
       "      <td>10.729112</td>\n",
       "      <td>770</td>\n",
       "      <td>0.0067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.781851</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.024969</td>\n",
       "      <td>0.408548</td>\n",
       "      <td>11</td>\n",
       "      <td>0.005223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.764291</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.848011</td>\n",
       "      <td>3.078016</td>\n",
       "      <td>422</td>\n",
       "      <td>0.001179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.985189</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.531435</td>\n",
       "      <td>0.030573</td>\n",
       "      <td>12</td>\n",
       "      <td>0.000694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>1.075003</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.915325</td>\n",
       "      <td>0.176816</td>\n",
       "      <td>9</td>\n",
       "      <td>0.001501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>1.342204</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.708582</td>\n",
       "      <td>0.919993</td>\n",
       "      <td>73</td>\n",
       "      <td>0.000756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>2.283096</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.799555</td>\n",
       "      <td>0.688189</td>\n",
       "      <td>158</td>\n",
       "      <td>0.000905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>7.896229</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.028388</td>\n",
       "      <td>0.102685</td>\n",
       "      <td>282</td>\n",
       "      <td>0.083223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>0.616272</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.235936</td>\n",
       "      <td>3.825898</td>\n",
       "      <td>64</td>\n",
       "      <td>0.032057</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>356 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     firing_rate  presence_ratio       snr  isi_violations_ratio  \\\n",
       "0        2.06766        0.983607  2.156246              5.724791   \n",
       "1       1.276477        0.897541  1.117945             10.729112   \n",
       "2       0.781851             1.0  4.024969              0.408548   \n",
       "3       1.764291             1.0  1.848011              3.078016   \n",
       "4       2.985189             1.0  3.531435              0.030573   \n",
       "..           ...             ...       ...                   ...   \n",
       "351     1.075003             1.0  4.915325              0.176816   \n",
       "352     1.342204             1.0  2.708582              0.919993   \n",
       "353     2.283096             1.0  2.799555              0.688189   \n",
       "354     7.896229             1.0  4.028388              0.102685   \n",
       "355     0.616272             1.0  1.235936              3.825898   \n",
       "\n",
       "     isi_violations_count  amplitude_cutoff  \n",
       "0                    1078           0.00309  \n",
       "1                     770            0.0067  \n",
       "2                      11          0.005223  \n",
       "3                     422          0.001179  \n",
       "4                      12          0.000694  \n",
       "..                    ...               ...  \n",
       "351                     9          0.001501  \n",
       "352                    73          0.000756  \n",
       "353                   158          0.000905  \n",
       "354                   282          0.083223  \n",
       "355                    64          0.032057  \n",
       "\n",
       "[356 rows x 6 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b10a113d-48de-4c2e-b2c6-7d0d43f5b326",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(amplitude_cutoff < 0.1) & (isi_violations_ratio < 1) & (presence_ratio > 0.9)\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd50c82f-6a55-486f-ba01-ce2be3081f1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f55ccd6-0014-400e-8d03-f4841f35bbb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c550b0b-d4e7-497e-8566-a24781dba18f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Emmett\\anaconda3\\envs\\si_env\\Lib\\site-packages\\spikeinterface\\core\\job_tools.py:103: UserWarning: `n_jobs` is not set so parallel processing is disabled! To speed up computations, it is recommended to set n_jobs either globally (with the `spikeinterface.set_global_job_kwargs()` function) or locally (with the `n_jobs` argument). Use `spikeinterface.set_global_job_kwargs?` for more information about job_kwargs.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export_report(): spike_amplitudes will not be exported. Use sorting_analyzer.compute('spike_amplitudes') if you want to include them.\n",
      "export_report(): correlograms will not be exported. Use sorting_anlyzer.compute('correlograms') if you want to include them.\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33f70ae0-3cdb-4249-9930-4b6e694d4c18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50a1a832-b5f3-46f3-a5dc-9d623a240684",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}