analyze_csnl_julia.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m   Updating\u001b[22m\u001b[39m registry at `~/.juliapro/JuliaPro_v1.3.1-1/registries/JuliaPro`\n",
      "######################################################################### 100.0%                         43.0%###                            65.0%################                           67.1%##########################         91.2%\n"
     ]
    },
    {
     "ename": "Base.IOError",
     "evalue": "IOError: unlink: permission denied (EACCES)",
     "output_type": "error",
     "traceback": [
      "IOError: unlink: permission denied (EACCES)",
      "",
      "Stacktrace:",
      " [1] unlink(::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [2] #rm#9(::Bool, ::Bool, ::typeof(rm), ::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [3] (::Base.Filesystem.var\"#kw##rm\")(::NamedTuple{(:force, :recursive),Tuple{Bool,Bool}}, ::typeof(rm), ::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [4] #rm#9(::Bool, ::Bool, ::typeof(rm), ::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [5] (::Base.Filesystem.var\"#kw##rm\")(::NamedTuple{(:force, :recursive),Tuple{Bool,Bool}}, ::typeof(rm), ::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [6] #rm#9(::Bool, ::Bool, ::typeof(rm), ::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [7] #checkfor_mv_cp_cptree#10(::Bool, ::typeof(Base.Filesystem.checkfor_mv_cp_cptree), ::String, ::String, ::String) at /Applications/JuliaPro-1.3.1-1.app/Contents/Resources/julia/Contents/Resources/julia/lib/julia/sys.dylib:?",
      " [8] #checkfor_mv_cp_cptree at ./none:0 [inlined]",
      " [9] #cp#12(::Bool, ::Bool, ::typeof(cp), ::String, ::String) at ./file.jl:338",
      " [10] (::Base.Filesystem.var\"#kw##cp\")(::NamedTuple{(:force,),Tuple{Bool}}, ::typeof(cp), ::String, ::String) at ./none:0",
      " [11] (::Pkg.Types.var\"#113#116\"{String})(::String) at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/Types.jl:1056",
      " [12] #mktempdir#18(::String, ::typeof(mktempdir), ::Pkg.Types.var\"#113#116\"{String}, ::String) at ./file.jl:634",
      " [13] mktempdir(::Function, ::String) at ./file.jl:632 (repeats 2 times)",
      " [14] #update_registries#111(::Bool, ::typeof(Pkg.Types.update_registries), ::Pkg.Types.Context, ::Array{Pkg.Types.RegistrySpec,1}) at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/Types.jl:1041",
      " [15] update_registries at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/Types.jl:1030 [inlined] (repeats 2 times)",
      " [16] #add#28(::Pkg.Types.PreserveLevel, ::Pkg.BinaryPlatforms.MacOS, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(Pkg.API.add), ::Pkg.Types.Context, ::Array{Pkg.Types.PackageSpec,1}) at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:143",
      " [17] add(::Pkg.Types.Context, ::Array{Pkg.Types.PackageSpec,1}) at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:112",
      " [18] #add#27 at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:109 [inlined]",
      " [19] add at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:109 [inlined]",
      " [20] #add#24 at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:107 [inlined]",
      " [21] add at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:107 [inlined]",
      " [22] #add#23(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(Pkg.API.add), ::String) at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:106",
      " [23] add(::String) at /Users/vagrant/worker/juliapro-release-osx1011-0_6/build/tmp_julia/Julia-1.3.app/Contents/Resources/julia/share/julia/stdlib/v1.3/Pkg/src/API.jl:106",
      " [24] top-level scope at In[2]:1"
     ]
    }
   ],
   "source": [
    "import Pkg; Pkg.add(\"Pkg\")\n",
    "import Pkg; Pkg.add(\"CSV\")\n",
    "import Pkg; Pkg.add(\"DataFrames\")\n",
    "import Pkg; Pkg.add(\"Statistics\")\n",
    "import Pkg; Pkg.add(\"DataFramesMeta\")\n",
    "#import Pkg; Pkg.add(\"Plots\")\n",
    "import Pkg; Pkg.add(\"StatsPlots\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "ArgumentError",
     "evalue": "ArgumentError: Package StatsPlots not found in current path:\n- Run `import Pkg; Pkg.add(\"StatsPlots\")` to install the StatsPlots package.\n",
     "output_type": "error",
     "traceback": [
      "ArgumentError: Package StatsPlots not found in current path:\n- Run `import Pkg; Pkg.add(\"StatsPlots\")` to install the StatsPlots package.\n",
      "",
      "Stacktrace:",
      " [1] require(::Module, ::Symbol) at ./loading.jl:887",
      " [2] top-level scope at In[5]:1"
     ]
    }
   ],
   "source": [
    "using CSV, DataFrames, Statistics, DataFramesMeta, StatsPlots #equivalent to pandas in python for dataframe operations and csv loading\n",
    "add https://github.com/giordano/FFMPEG.jl\n",
    "csnl_data = CSV.read(\"cnsl_data.csv\")\n",
    "rename!(csnl_data, Dict(1 => :sample)) #add a label \"sample\" to the first column for clarity\n",
    "csnl_long = stack(csnl_data,2:102)\n",
    "CSV.write(\"cnsl_long.csv\",csnl_long)\n",
    "\n",
    "csnlnames= names(csnl_data)\n",
    "ethnicities = unique(csnl_data.ethnicity)\n",
    "samples = csnl_data[!,1]\n",
    "csnl_columns = csnlnames[3:52] #hard code the csnl gene probes vs non csnl\n",
    "noncsnl_columns = csnlnames[53:102]\n",
    "noncsnl_values = csnl_data[:,noncsnl_columns] #peel off the noncsnl read depths\n",
    "ncsnlv=convert(Matrix,noncsnl_values) #need to convert to matrix to get the mean\n",
    "non_csnl_norms = mean(ncsnlv, dims=2) #use the mean of all the non_csnl to get a normalization factor for each sample\n",
    "#csnl_data[noncsnl_columns]\n",
    "csnl_normalized = csnl_data[!,3:102]./non_csnl_norms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>ethnicity</th><th>avgprobe0</th><th>avgnon0</th></tr><tr><th></th><th>String</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>3 rows × 3 columns</p><tr><th>1</th><td>A</td><td>735.901</td><td>387.472</td></tr><tr><th>2</th><td>C</td><td>744.187</td><td>392.482</td></tr><tr><th>3</th><td>B</td><td>741.309</td><td>390.974</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& ethnicity & avgprobe0 & avgnon0\\\\\n",
       "\t\\hline\n",
       "\t& String & Float64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & A & 735.901 & 387.472 \\\\\n",
       "\t2 & C & 744.187 & 392.482 \\\\\n",
       "\t3 & B & 741.309 & 390.974 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "3×3 DataFrame\n",
       "│ Row │ ethnicity │ avgprobe0 │ avgnon0 │\n",
       "│     │ \u001b[90mString\u001b[39m    │ \u001b[90mFloat64\u001b[39m   │ \u001b[90mFloat64\u001b[39m │\n",
       "├─────┼───────────┼───────────┼─────────┤\n",
       "│ 1   │ A         │ 735.901   │ 387.472 │\n",
       "│ 2   │ C         │ 744.187   │ 392.482 │\n",
       "│ 3   │ B         │ 741.309   │ 390.974 │"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by(csnl_data, :ethnicity, avgprobe0 = :CNSL_probe_0 => mean, avgnon0 = :non_CNSL_probe_0 => mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "filteredDf = csnl_data[startswith.(df.field,pattern),:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102-element Array{Symbol,1}:\n",
       " :sample           \n",
       " :ethnicity        \n",
       " :CNSL_probe_0     \n",
       " :CNSL_probe_1     \n",
       " :CNSL_probe_2     \n",
       " :CNSL_probe_3     \n",
       " :CNSL_probe_4     \n",
       " :CNSL_probe_5     \n",
       " :CNSL_probe_6     \n",
       " :CNSL_probe_7     \n",
       " :CNSL_probe_8     \n",
       " :CNSL_probe_9     \n",
       " :CNSL_probe_10    \n",
       " ⋮                 \n",
       " :non_CNSL_probe_38\n",
       " :non_CNSL_probe_39\n",
       " :non_CNSL_probe_40\n",
       " :non_CNSL_probe_41\n",
       " :non_CNSL_probe_42\n",
       " :non_CNSL_probe_43\n",
       " :non_CNSL_probe_44\n",
       " :non_CNSL_probe_45\n",
       " :non_CNSL_probe_46\n",
       " :non_CNSL_probe_47\n",
       " :non_CNSL_probe_48\n",
       " :non_CNSL_probe_49"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csnlnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "#amask = csnl_data.ethnicity.==\"A\"\n",
    "#bmask = csnl_data.ethnicity.==\"B\"\n",
    "#cmask = csnl_data.ethnicity.==\"C\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m  Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  Installed\u001b[22m\u001b[39m NormalizeQuantiles ─ v1.0.0\n",
      "\u001b[32m\u001b[1m   Updating\u001b[22m\u001b[39m `~/.juliapro/JuliaPro_v1.3.1-1/environments/v1.3/Project.toml`\n",
      "\u001b[90m   d366530f \u001b[39m\u001b[92m+ NormalizeQuantiles v1.0.0\u001b[39m\n",
      "\u001b[32m\u001b[1m   Updating\u001b[22m\u001b[39m `~/.juliapro/JuliaPro_v1.3.1-1/environments/v1.3/Manifest.toml`\n",
      "\u001b[90m   d366530f \u001b[39m\u001b[92m+ NormalizeQuantiles v1.0.0\u001b[39m\n",
      "┌ Info: Precompiling NormalizeQuantiles [d366530f-4af1-5ff1-9928-73bf22b88830]\n",
      "└ @ Base loading.jl:1273\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dict{Int64,Array{Int64,N} where N} with 4 entries:\n",
       "  4 => [4, 7, 8]\n",
       "  2 => [2, 6]\n",
       "  3 => [3]\n",
       "  1 => [1, 5]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Pkg.add(\"NormalizeQuantiles\")\n",
    "using NormalizeQuantiles\n",
    "A = [1,2,3,4,1,2,4,4];\n",
    "(r,m) = sampleRanks(A,resultMatrix=true);\n",
    "m"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.1",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}