run analysis, rename folders

1830416002 · Apr 16, 2021 · 4db6a79 · 4db6a79
1 parent 4b057fb
commit 4db6a79
Show file tree

Hide file tree

Showing 39 changed files with 54 additions and 162 deletions.
diff --git a/README.md b/README.md
@@ -153,19 +153,23 @@ To get a local copy up and running follow these simple steps.
    export PYTHONPATH=~/pnet_prostate_paper:$PYTHONPATH
    ```
 
-3. To train all models run (time consuming process) 
-   ```sh
-   cd ./run
-   python run_me.py
+3. To generate all paper figures, run 
+     ```sh
+   cd ./analysis
+   python run_it_all.py
    ```
-4. To generate paper figures run the different files under the 'analysis' directory, e.g. 
+
+4. To generate individual paper figure run the different files under the 'analysis' directory, e.g. 
      ```sh
    cd ./analysis
    python figure_1_d_auc_prc.py
    ```
-
-
-
+
+5. To re-train all models from scratch run (time consuming process) 
+   ```sh
+   cd ./train
+   python run_me.py
+   ```
 
 
 <!-- LICENSE -->

diff --git a/analysis/figure_3/figure3_b_gene_importance.py b/analysis/figure_3/figure3_b_gene_importance.py
@@ -8,6 +8,8 @@
 from os.path import join, dirname, realpath, exists
 from os import makedirs
 
+module_path = dirname(realpath(__file__))
+
 def plot_high_genes_sns(df, col='avg_score', name='', saving_directory='.'):
     df.index=df.index.map(shorten_names)
     x_pos = range(df.shape[0])
@@ -239,12 +241,12 @@ def plot_high_genes2(ax, layer=1, graph ='hist', direction='h'):
     else:
         column = 'coef'
 
-    node_importance = pd.read_csv('extracted/node_importance_graph_adjusted.csv', index_col=0)
+    node_importance = pd.read_csv(join(module_path,'./extracted/node_importance_graph_adjusted.csv'), index_col=0)
     high_nodes = node_importance[node_importance.layer == layer].abs().nlargest(10, columns=[column])
     # high_nodes = node_importance[node_importance.layer == layer].abs().nlargest(10, columns=['coef'])
     features = list(high_nodes.index)
-    response = pd.read_csv('extracted/response.csv', index_col=0)
-    df_in = pd.read_csv('./extracted/gradient_importance_detailed_{}.csv'.format(layer), index_col=0)
+    response = pd.read_csv(join(module_path,'./extracted/response.csv'), index_col=0)
+    df_in = pd.read_csv(join(module_path, './extracted/gradient_importance_detailed_{}.csv').format(layer), index_col=0)
     df_in = df_in.copy()
     df_in = df_in.join(response)
     df_in['group'] = df_in.response
@@ -365,11 +367,11 @@ def shorten_names(name):
         name= name[:60]+'...'
     return name
 
-current_dir = dirname(realpath(__file__))
+
 
 def run():
-    node_importance = pd.read_csv(join(current_dir,'extracted/node_importance_graph_adjusted.csv'), index_col=0)
-    response = pd.read_csv(join(current_dir, 'extracted/response.csv'), index_col=0)
+    node_importance = pd.read_csv(join(module_path,'extracted/node_importance_graph_adjusted.csv'), index_col=0)
+    response = pd.read_csv(join(module_path, 'extracted/response.csv'), index_col=0)
     print response.head()
     layers = list(node_importance.layer.unique())
     print layers

diff --git a/analysis/figure_3/figure_3_sankey_all.py b/analysis/figure_3/figure_3_sankey_all.py
@@ -88,11 +88,11 @@ def get_nodes_per_layer_filtered(nodes_per_layer_df, all_node_ids, all_node_labe
     nodes_per_layer_filtered_df.index = nodes_per_layer_filtered_df.index.map(lambda x: mapping_dict[x])
     return nodes_per_layer_filtered_df
 
-features_weights = pd.read_csv(join(module_path,'./extracted/gradient_importance_0.csv'), index_col =[0,1])
-features_weights = features_weights.reset_index()
-features_weights.columns= ['target', 'source', 'value']
-features_weights['layer'] = 0
-features_weights.head()
+# features_weights = pd.read_csv(join(module_path,'./extracted/gradient_importance_0.csv'), index_col =[0,1])
+# features_weights = features_weights.reset_index()
+# features_weights.columns= ['target', 'source', 'value']
+# features_weights['layer'] = 0
+# features_weights.head()
 
 def get_links_with_first_layer():
     '''

diff --git a/config_path.py b/config_path.py
@@ -6,13 +6,10 @@
 PATHWAY_PATH = join(DATA_PATH, 'pathways')
 REACTOM_PATHWAY_PATH = join(PATHWAY_PATH, 'Reactome')
 PROSTATE_DATA_PATH = join(DATA_PATH, 'prostate')
-MELANOMA_DATA_PATH = join(DATA_PATH, 'melanoma')
 RUN_PATH = join(BASE_PATH, 'run')
 LOG_PATH = join(BASE_PATH, '_logs')
 PROSTATE_LOG_PATH = join(LOG_PATH, 'p1000')
-MELANOMA_LOG_PATH = join(LOG_PATH, 'melanoma')
 PARAMS_PATH = join(RUN_PATH, 'params')
 POSTATE_PARAMS_PATH = join(PARAMS_PATH, 'P1000')
-MELANOMA_PARAMS_PATH = join(PARAMS_PATH, 'melanoma')
 PLOTS_PATH = join(BASE_PATH, '_plots')
 
diff --git a/run/run_me_melanoma.py b/run/run_me_melanoma.py
diff --git a/run/__init__.py → train/__init__.py b/run/__init__.py → train/__init__.py
diff --git a/...ssvalidation_ML_params_search_logistic.py → ...ssvalidation_ML_params_search_logistic.py b/...ssvalidation_ML_params_search_logistic.py → ...ssvalidation_ML_params_search_logistic.py
diff --git a/...h/crossvalidation_ML_params_search_svm.py → ...h/crossvalidation_ML_params_search_svm.py b/...h/crossvalidation_ML_params_search_svm.py → ...h/crossvalidation_ML_params_search_svm.py
diff --git a/...rch/onesplit_ML_params_search_adaboost.py → ...rch/onesplit_ML_params_search_adaboost.py b/...rch/onesplit_ML_params_search_adaboost.py → ...rch/onesplit_ML_params_search_adaboost.py
diff --git a/...onesplit_ML_params_search_randomforest.py → ...onesplit_ML_params_search_randomforest.py b/...onesplit_ML_params_search_randomforest.py → ...onesplit_ML_params_search_randomforest.py
diff --git a/...s_search/onesplit_ML_params_search_svm.py → ...s_search/onesplit_ML_params_search_svm.py b/...s_search/onesplit_ML_params_search_svm.py → ...s_search/onesplit_ML_params_search_svm.py
diff --git a/...h/onesplit_ML_params_search_svm_linear.py → ...h/onesplit_ML_params_search_svm_linear.py b/...h/onesplit_ML_params_search_svm_linear.py → ...h/onesplit_ML_params_search_svm_linear.py
diff --git a/..._search/onesplit_ML_params_search_tree.py → ..._search/onesplit_ML_params_search_tree.py b/..._search/onesplit_ML_params_search_tree.py → ..._search/onesplit_ML_params_search_tree.py
diff --git a/...arch/onsplit_ML_params_search_logistic.py → ...arch/onsplit_ML_params_search_logistic.py b/...arch/onsplit_ML_params_search_logistic.py → ...arch/onsplit_ML_params_search_logistic.py
diff --git a/.../P1000/compare/crossvalidation_ML_test.py → .../P1000/compare/crossvalidation_ML_test.py b/.../P1000/compare/crossvalidation_ML_test.py → .../P1000/compare/crossvalidation_ML_test.py
diff --git a/run/params/P1000/compare/onsplit_ML_test.py → ...n/params/P1000/compare/onsplit_ML_test.py b/run/params/P1000/compare/onsplit_ML_test.py → ...n/params/P1000/compare/onsplit_ML_test.py
diff --git a/...split_number_samples_dense_sameweights.py → ...split_number_samples_dense_sameweights.py b/...split_number_samples_dense_sameweights.py → ...split_number_samples_dense_sameweights.py
diff --git a/run/params/P1000/dense/onsplit_dense.py → train/params/P1000/dense/onsplit_dense.py b/run/params/P1000/dense/onsplit_dense.py → train/params/P1000/dense/onsplit_dense.py
diff --git a/...00/external_validation/pnet_validation.py → ...00/external_validation/pnet_validation.py b/...00/external_validation/pnet_validation.py → ...00/external_validation/pnet_validation.py
diff --git a/...samples/crossvalidation_average_reg_10.py → ...samples/crossvalidation_average_reg_10.py b/...samples/crossvalidation_average_reg_10.py → ...samples/crossvalidation_average_reg_10.py
diff --git a/...es/crossvalidation_average_reg_10_tanh.py → ...es/crossvalidation_average_reg_10_tanh.py b/...es/crossvalidation_average_reg_10_tanh.py → ...es/crossvalidation_average_reg_10_tanh.py
diff --git a/...ation_number_samples_dense_sameweights.py → ...ation_number_samples_dense_sameweights.py b/...ation_number_samples_dense_sameweights.py → ...ation_number_samples_dense_sameweights.py
diff --git a/...et/crossvalidation_average_reg_10_tanh.py → ...et/crossvalidation_average_reg_10_tanh.py b/...et/crossvalidation_average_reg_10_tanh.py → ...et/crossvalidation_average_reg_10_tanh.py
diff --git a/...plit_average_reg_10_tanh_large_testing.py → ...plit_average_reg_10_tanh_large_testing.py b/...plit_average_reg_10_tanh_large_testing.py → ...plit_average_reg_10_tanh_large_testing.py
diff --git a/...verage_reg_10_tanh_large_testing_inner.py → ...verage_reg_10_tanh_large_testing_inner.py b/...verage_reg_10_tanh_large_testing_inner.py → ...verage_reg_10_tanh_large_testing_inner.py
diff --git a/...1000/review/10custom_arch/onsplit_kegg.py → ...1000/review/10custom_arch/onsplit_kegg.py b/...1000/review/10custom_arch/onsplit_kegg.py → ...1000/review/10custom_arch/onsplit_kegg.py
diff --git a/...verage_reg_10_tanh_large_testing_count.py → ...verage_reg_10_tanh_large_testing_count.py b/...verage_reg_10_tanh_large_testing_count.py → ...verage_reg_10_tanh_large_testing_count.py
diff --git a/...rage_reg_10_tanh_large_testing_hotspot.py → ...rage_reg_10_tanh_large_testing_hotspot.py b/...rage_reg_10_tanh_large_testing_hotspot.py → ...rage_reg_10_tanh_large_testing_hotspot.py
diff --git a/...dation_average_reg_10_tanh_single_copy.py → ...dation_average_reg_10_tanh_single_copy.py b/...dation_average_reg_10_tanh_single_copy.py → ...dation_average_reg_10_tanh_single_copy.py
diff --git a/..._reg_10_tanh_large_testing_single_copy.py → ..._reg_10_tanh_large_testing_single_copy.py b/..._reg_10_tanh_large_testing_single_copy.py → ..._reg_10_tanh_large_testing_single_copy.py
diff --git a/run/params/P1000/review/LOOCV_reg_10_tanh.py → .../params/P1000/review/LOOCV_reg_10_tanh.py b/run/params/P1000/review/LOOCV_reg_10_tanh.py → .../params/P1000/review/LOOCV_reg_10_tanh.py
diff --git a/...ation_average_reg_10_tanh_cancer_genes.py → ...ation_average_reg_10_tanh_cancer_genes.py b/...ation_average_reg_10_tanh_cancer_genes.py → ...ation_average_reg_10_tanh_cancer_genes.py
diff --git a/..._average_reg_10_tanh_large_testing_TMB.py → ..._average_reg_10_tanh_large_testing_TMB.py b/..._average_reg_10_tanh_large_testing_TMB.py → ..._average_reg_10_tanh_large_testing_TMB.py
diff --git a/...erage_reg_10_tanh_large_testing_fusion.py → ...erage_reg_10_tanh_large_testing_fusion.py b/...erage_reg_10_tanh_large_testing_fusion.py → ...erage_reg_10_tanh_large_testing_fusion.py
diff --git a/..._reg_10_tanh_large_testing_fusion_zero.py → ..._reg_10_tanh_large_testing_fusion_zero.py b/..._reg_10_tanh_large_testing_fusion_zero.py → ..._reg_10_tanh_large_testing_fusion_zero.py
diff --git a/..._tanh_large_testing_inner_fusion_genes.py → ..._tanh_large_testing_inner_fusion_genes.py b/..._tanh_large_testing_inner_fusion_genes.py → ..._tanh_large_testing_inner_fusion_genes.py
diff --git a/...age_reg_10_tanh_large_testing_inner_LR.py → ...age_reg_10_tanh_large_testing_inner_LR.py b/...age_reg_10_tanh_large_testing_inner_LR.py → ...age_reg_10_tanh_large_testing_inner_LR.py
diff --git a/...t_average_reg_10_tanh_large_testing_ge.py → ...t_average_reg_10_tanh_large_testing_ge.py b/...t_average_reg_10_tanh_large_testing_ge.py → ...t_average_reg_10_tanh_large_testing_ge.py
diff --git a/run/run_me.py → train/run_me.py b/run/run_me.py → train/run_me.py
@@ -44,51 +44,55 @@ def elapsed_time(start_time, end_time):
 params_file_list.append('./pnet/onsplit_average_reg_10_tanh_large_testing')
 params_file_list.append('./pnet/onsplit_average_reg_10_tanh_large_testing_inner')
 params_file_list.append('./pnet/crossvalidation_average_reg_10_tanh')
-
-# other ML models
+#
+# # other ML models
 params_file_list.append('./compare/onsplit_ML_test')
 params_file_list.append('./compare/crossvalidation_ML_test')
-
-# dense
+#
+# # dense
 params_file_list.append('./dense/onesplit_number_samples_dense_sameweights')
 params_file_list.append('./dense/onsplit_dense')
-
-# number_samples
+#
+# # number_samples
 params_file_list.append('./number_samples/crossvalidation_average_reg_10')
 ## params_file_list.append('./number_samples/crossvalidation_average_reg_10_tanh')
 params_file_list.append('./number_samples/crossvalidation_number_samples_dense_sameweights')
-
-# external_validation
+#
+# # external_validation
 params_file_list.append('./external_validation/pnet_validation')
-
-#reviews------------------------------------
-#LOOCV
-params_file_list.append('./review/LOOCV_reg_10_tanh')
-#ge
+#
+# #reviews------------------------------------
+# #LOOCV
+# params_file_list.append('./review/LOOCV_reg_10_tanh')
+# #ge
 params_file_list.append('./review/onsplit_average_reg_10_tanh_large_testing_ge')
-#fusion
-params_file_list.append('./review/fusion/onsplit_average_reg_10_tanh_large_testing_TMB')
+# #fusion
+# params_file_list.append('./review/fusion/onsplit_average_reg_10_tanh_large_testing_TMB')
 params_file_list.append('./review/fusion/onsplit_average_reg_10_tanh_large_testing_fusion')
 params_file_list.append('./review/fusion/onsplit_average_reg_10_tanh_large_testing_fusion_zero')
 params_file_list.append('./review/fusion/onsplit_average_reg_10_tanh_large_testing_inner_fusion_genes')
-
-#single copy
+#
+# #single copy
 params_file_list.append('./review/9single_copy/onsplit_average_reg_10_tanh_large_testing_single_copy')
 params_file_list.append('./review/9single_copy/crossvalidation_average_reg_10_tanh_single_copy')
-
-#custom arch
+#
+# #custom arch
 params_file_list.append('./review/10custom_arch/onsplit_kegg')
-
-#learning rate
+#
+# #learning rate
 params_file_list.append('./review/learning_rate/onsplit_average_reg_10_tanh_large_testing_inner_LR')
 
 
+#hotspot
+params_file_list.append('./review/9hotspot/onsplit_average_reg_10_tanh_large_testing_hotspot')
+params_file_list.append('./review/9hotspot/onsplit_average_reg_10_tanh_large_testing_count')
+
 #cancer genes
-# params_file_list.append('./review/onsplit_average_reg_10_tanh_large_testing')
-# params_file_list.append('./review/onsplit_average_reg_10_cancer_genes_testing')
-# params_file_list.append('./review/crossvalidation_average_reg_10_tanh_cancer_genes')
+params_file_list.append('./review/onsplit_average_reg_10_tanh_large_testing')
+params_file_list.append('./review/onsplit_average_reg_10_cancer_genes_testing')
+params_file_list.append('./review/crossvalidation_average_reg_10_tanh_cancer_genes')
+
 
-# melanoma