From bb764c75b2692f0a5ca393710acf8ded353c98e8 Mon Sep 17 00:00:00 2001
From: Daniel Himmelstein <daniel.himmelstein@gmail.com>
Date: Wed, 21 Sep 2016 11:35:21 -0400
Subject: [PATCH] Save grid_search performance metrics

---
 cognoml/analysis.py        |    7 +-
 data/api/hippo-output.json | 1263 ++++++++++++++++++++++++++++++++++++
 2 files changed, 1269 insertions(+), 1 deletion(-)

diff --git a/cognoml/analysis.py b/cognoml/analysis.py
index 4906e39..f51904c 100644
--- a/cognoml/analysis.py
+++ b/cognoml/analysis.py
@@ -54,7 +54,6 @@ def classify(sample_id, mutation_status, **kwargs):
     obs_df['testing'] = obs_df.sample_id.isin(X_test.index).astype(int)
     
     pipeline.fit(X=X_train, y=y_train)
-    #cv_score_df = grid_scores_to_df(clf_grid.grid_scores_)
     
     predict_df = pd.DataFrame.from_items([
         ('sample_id', X_whole.index),
@@ -95,6 +94,12 @@ def classify(sample_id, mutation_status, **kwargs):
     performance['cv'] = {'auroc': round(clf_grid.best_score_, 5)}
     results['performance'] = performance
     
+    gs = collections.OrderedDict()
+    gs['mean_scores'] = utils.mean_grid_scores_to_df(clf_grid.grid_scores_)
+    gs['fold_scores'] = utils.grid_scores_to_df(clf_grid.grid_scores_)
+    gs = utils.value_map(gs, utils.df_to_datatables)
+    results['grid_search'] = gs
+    
     results['model'] = utils.model_info(clf_grid.best_estimator_)
 
     feature_df = utils.get_feature_df(clf_grid.best_estimator_, X.columns)
diff --git a/data/api/hippo-output.json b/data/api/hippo-output.json
index 48dddce..1d07303 100644
--- a/data/api/hippo-output.json
+++ b/data/api/hippo-output.json
@@ -32,6 +32,1269 @@
       "auroc": 0.62524
     }
   },
+  "grid_search": {
+    "fold_scores": {
+      "columns": [
+        "alpha",
+        "fold",
+        "l1_ratio",
+        "score"
+      ],
+      "data": [
+        [
+          0.0001,
+          0,
+          0.0,
+          0.48733
+        ],
+        [
+          0.0001,
+          1,
+          0.0,
+          0.59161
+        ],
+        [
+          0.0001,
+          2,
+          0.0,
+          0.59526
+        ],
+        [
+          0.0001,
+          0,
+          0.05,
+          0.49873
+        ],
+        [
+          0.0001,
+          1,
+          0.05,
+          0.58957
+        ],
+        [
+          0.0001,
+          2,
+          0.05,
+          0.59915
+        ],
+        [
+          0.0001,
+          0,
+          0.1,
+          0.50386
+        ],
+        [
+          0.0001,
+          1,
+          0.1,
+          0.58154
+        ],
+        [
+          0.0001,
+          2,
+          0.1,
+          0.59797
+        ],
+        [
+          0.0001,
+          0,
+          0.2,
+          0.50539
+        ],
+        [
+          0.0001,
+          1,
+          0.2,
+          0.5915
+        ],
+        [
+          0.0001,
+          2,
+          0.2,
+          0.5942
+        ],
+        [
+          0.0001,
+          0,
+          0.5,
+          0.5232
+        ],
+        [
+          0.0001,
+          1,
+          0.5,
+          0.59996
+        ],
+        [
+          0.0001,
+          2,
+          0.5,
+          0.57353
+        ],
+        [
+          0.0001,
+          0,
+          0.8,
+          0.50881
+        ],
+        [
+          0.0001,
+          1,
+          0.8,
+          0.5889
+        ],
+        [
+          0.0001,
+          2,
+          0.8,
+          0.57541
+        ],
+        [
+          0.0001,
+          0,
+          0.9,
+          0.51105
+        ],
+        [
+          0.0001,
+          1,
+          0.9,
+          0.57999
+        ],
+        [
+          0.0001,
+          2,
+          0.9,
+          0.58455
+        ],
+        [
+          0.0001,
+          0,
+          0.95,
+          0.52634
+        ],
+        [
+          0.0001,
+          1,
+          0.95,
+          0.5814
+        ],
+        [
+          0.0001,
+          2,
+          0.95,
+          0.58795
+        ],
+        [
+          0.0001,
+          0,
+          1.0,
+          0.52799
+        ],
+        [
+          0.0001,
+          1,
+          1.0,
+          0.58302
+        ],
+        [
+          0.0001,
+          2,
+          1.0,
+          0.58462
+        ],
+        [
+          0.001,
+          0,
+          0.0,
+          0.53017
+        ],
+        [
+          0.001,
+          1,
+          0.0,
+          0.59751
+        ],
+        [
+          0.001,
+          2,
+          0.0,
+          0.62112
+        ],
+        [
+          0.001,
+          0,
+          0.05,
+          0.54779
+        ],
+        [
+          0.001,
+          1,
+          0.05,
+          0.59732
+        ],
+        [
+          0.001,
+          2,
+          0.05,
+          0.62748
+        ],
+        [
+          0.001,
+          0,
+          0.1,
+          0.57378
+        ],
+        [
+          0.001,
+          1,
+          0.1,
+          0.60776
+        ],
+        [
+          0.001,
+          2,
+          0.1,
+          0.62781
+        ],
+        [
+          0.001,
+          0,
+          0.2,
+          0.58603
+        ],
+        [
+          0.001,
+          1,
+          0.2,
+          0.6182
+        ],
+        [
+          0.001,
+          2,
+          0.2,
+          0.62887
+        ],
+        [
+          0.001,
+          0,
+          0.5,
+          0.59417
+        ],
+        [
+          0.001,
+          1,
+          0.5,
+          0.62006
+        ],
+        [
+          0.001,
+          2,
+          0.5,
+          0.6283
+        ],
+        [
+          0.001,
+          0,
+          0.8,
+          0.59471
+        ],
+        [
+          0.001,
+          1,
+          0.8,
+          0.61823
+        ],
+        [
+          0.001,
+          2,
+          0.8,
+          0.62853
+        ],
+        [
+          0.001,
+          0,
+          0.9,
+          0.59497
+        ],
+        [
+          0.001,
+          1,
+          0.9,
+          0.61808
+        ],
+        [
+          0.001,
+          2,
+          0.9,
+          0.62852
+        ],
+        [
+          0.001,
+          0,
+          0.95,
+          0.59507
+        ],
+        [
+          0.001,
+          1,
+          0.95,
+          0.61803
+        ],
+        [
+          0.001,
+          2,
+          0.95,
+          0.62852
+        ],
+        [
+          0.001,
+          0,
+          1.0,
+          0.59516
+        ],
+        [
+          0.001,
+          1,
+          1.0,
+          0.61811
+        ],
+        [
+          0.001,
+          2,
+          1.0,
+          0.62852
+        ],
+        [
+          0.01,
+          0,
+          0.0,
+          0.60689
+        ],
+        [
+          0.01,
+          1,
+          0.0,
+          0.62088
+        ],
+        [
+          0.01,
+          2,
+          0.0,
+          0.63339
+        ],
+        [
+          0.01,
+          0,
+          0.05,
+          0.6124
+        ],
+        [
+          0.01,
+          1,
+          0.05,
+          0.62786
+        ],
+        [
+          0.01,
+          2,
+          0.05,
+          0.63128
+        ],
+        [
+          0.01,
+          0,
+          0.1,
+          0.61214
+        ],
+        [
+          0.01,
+          1,
+          0.1,
+          0.62839
+        ],
+        [
+          0.01,
+          2,
+          0.1,
+          0.63114
+        ],
+        [
+          0.01,
+          0,
+          0.2,
+          0.61218
+        ],
+        [
+          0.01,
+          1,
+          0.2,
+          0.62935
+        ],
+        [
+          0.01,
+          2,
+          0.2,
+          0.63102
+        ],
+        [
+          0.01,
+          0,
+          0.5,
+          0.61277
+        ],
+        [
+          0.01,
+          1,
+          0.5,
+          0.62914
+        ],
+        [
+          0.01,
+          2,
+          0.5,
+          0.63054
+        ],
+        [
+          0.01,
+          0,
+          0.8,
+          0.61221
+        ],
+        [
+          0.01,
+          1,
+          0.8,
+          0.63205
+        ],
+        [
+          0.01,
+          2,
+          0.8,
+          0.63027
+        ],
+        [
+          0.01,
+          0,
+          0.9,
+          0.61144
+        ],
+        [
+          0.01,
+          1,
+          0.9,
+          0.63253
+        ],
+        [
+          0.01,
+          2,
+          0.9,
+          0.62988
+        ],
+        [
+          0.01,
+          0,
+          0.95,
+          0.61116
+        ],
+        [
+          0.01,
+          1,
+          0.95,
+          0.6326
+        ],
+        [
+          0.01,
+          2,
+          0.95,
+          0.62974
+        ],
+        [
+          0.01,
+          0,
+          1.0,
+          0.61098
+        ],
+        [
+          0.01,
+          1,
+          1.0,
+          0.62954
+        ],
+        [
+          0.01,
+          2,
+          1.0,
+          0.6295
+        ],
+        [
+          0.1,
+          0,
+          0.0,
+          0.61255
+        ],
+        [
+          0.1,
+          1,
+          0.0,
+          0.62922
+        ],
+        [
+          0.1,
+          2,
+          0.0,
+          0.63395
+        ],
+        [
+          0.1,
+          0,
+          0.05,
+          0.61279
+        ],
+        [
+          0.1,
+          1,
+          0.05,
+          0.62893
+        ],
+        [
+          0.1,
+          2,
+          0.05,
+          0.63227
+        ],
+        [
+          0.1,
+          0,
+          0.1,
+          0.61169
+        ],
+        [
+          0.1,
+          1,
+          0.1,
+          0.62912
+        ],
+        [
+          0.1,
+          2,
+          0.1,
+          0.63152
+        ],
+        [
+          0.1,
+          0,
+          0.2,
+          0.60237
+        ],
+        [
+          0.1,
+          1,
+          0.2,
+          0.62988
+        ],
+        [
+          0.1,
+          2,
+          0.2,
+          0.62192
+        ],
+        [
+          0.1,
+          0,
+          0.5,
+          0.57488
+        ],
+        [
+          0.1,
+          1,
+          0.5,
+          0.61841
+        ],
+        [
+          0.1,
+          2,
+          0.5,
+          0.60419
+        ],
+        [
+          0.1,
+          0,
+          0.8,
+          0.5
+        ],
+        [
+          0.1,
+          1,
+          0.8,
+          0.5
+        ],
+        [
+          0.1,
+          2,
+          0.8,
+          0.5
+        ],
+        [
+          0.1,
+          0,
+          0.9,
+          0.5
+        ],
+        [
+          0.1,
+          1,
+          0.9,
+          0.5
+        ],
+        [
+          0.1,
+          2,
+          0.9,
+          0.5
+        ],
+        [
+          0.1,
+          0,
+          0.95,
+          0.5
+        ],
+        [
+          0.1,
+          1,
+          0.95,
+          0.5
+        ],
+        [
+          0.1,
+          2,
+          0.95,
+          0.5
+        ],
+        [
+          0.1,
+          0,
+          1.0,
+          0.5
+        ],
+        [
+          0.1,
+          1,
+          1.0,
+          0.5
+        ],
+        [
+          0.1,
+          2,
+          1.0,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          0.0,
+          0.60965
+        ],
+        [
+          1.0,
+          1,
+          0.0,
+          0.62621
+        ],
+        [
+          1.0,
+          2,
+          0.0,
+          0.6302
+        ],
+        [
+          1.0,
+          0,
+          0.05,
+          0.57488
+        ],
+        [
+          1.0,
+          1,
+          0.05,
+          0.56982
+        ],
+        [
+          1.0,
+          2,
+          0.05,
+          0.60419
+        ],
+        [
+          1.0,
+          0,
+          0.1,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          0.1,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          0.1,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          0.2,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          0.2,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          0.2,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          0.5,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          0.5,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          0.5,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          0.8,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          0.8,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          0.8,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          0.9,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          0.9,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          0.9,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          0.95,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          0.95,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          0.95,
+          0.5
+        ],
+        [
+          1.0,
+          0,
+          1.0,
+          0.5
+        ],
+        [
+          1.0,
+          1,
+          1.0,
+          0.5
+        ],
+        [
+          1.0,
+          2,
+          1.0,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.0,
+          0.6079
+        ],
+        [
+          10.0,
+          1,
+          0.0,
+          0.62431
+        ],
+        [
+          10.0,
+          2,
+          0.0,
+          0.62759
+        ],
+        [
+          10.0,
+          0,
+          0.05,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.05,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.05,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.1,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.1,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.1,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.2,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.2,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.2,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.5,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.5,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.5,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.8,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.8,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.8,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.9,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.9,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.9,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          0.95,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          0.95,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          0.95,
+          0.5
+        ],
+        [
+          10.0,
+          0,
+          1.0,
+          0.5
+        ],
+        [
+          10.0,
+          1,
+          1.0,
+          0.5
+        ],
+        [
+          10.0,
+          2,
+          1.0,
+          0.5
+        ]
+      ]
+    },
+    "mean_scores": {
+      "columns": [
+        "alpha",
+        "l1_ratio",
+        "score"
+      ],
+      "data": [
+        [
+          0.0001,
+          0.0,
+          0.55805
+        ],
+        [
+          0.0001,
+          0.05,
+          0.56247
+        ],
+        [
+          0.0001,
+          0.1,
+          0.56111
+        ],
+        [
+          0.0001,
+          0.2,
+          0.56368
+        ],
+        [
+          0.0001,
+          0.5,
+          0.56555
+        ],
+        [
+          0.0001,
+          0.8,
+          0.55769
+        ],
+        [
+          0.0001,
+          0.9,
+          0.55852
+        ],
+        [
+          0.0001,
+          0.95,
+          0.56522
+        ],
+        [
+          0.0001,
+          1.0,
+          0.5652
+        ],
+        [
+          0.001,
+          0.0,
+          0.58292
+        ],
+        [
+          0.001,
+          0.05,
+          0.59085
+        ],
+        [
+          0.001,
+          0.1,
+          0.60311
+        ],
+        [
+          0.001,
+          0.2,
+          0.61103
+        ],
+        [
+          0.001,
+          0.5,
+          0.61417
+        ],
+        [
+          0.001,
+          0.8,
+          0.61382
+        ],
+        [
+          0.001,
+          0.9,
+          0.61385
+        ],
+        [
+          0.001,
+          0.95,
+          0.61387
+        ],
+        [
+          0.001,
+          1.0,
+          0.61392
+        ],
+        [
+          0.01,
+          0.0,
+          0.62038
+        ],
+        [
+          0.01,
+          0.05,
+          0.62385
+        ],
+        [
+          0.01,
+          0.1,
+          0.62389
+        ],
+        [
+          0.01,
+          0.2,
+          0.62418
+        ],
+        [
+          0.01,
+          0.5,
+          0.62415
+        ],
+        [
+          0.01,
+          0.8,
+          0.62484
+        ],
+        [
+          0.01,
+          0.9,
+          0.62461
+        ],
+        [
+          0.01,
+          0.95,
+          0.6245
+        ],
+        [
+          0.01,
+          1.0,
+          0.62333
+        ],
+        [
+          0.1,
+          0.0,
+          0.62524
+        ],
+        [
+          0.1,
+          0.05,
+          0.62466
+        ],
+        [
+          0.1,
+          0.1,
+          0.62411
+        ],
+        [
+          0.1,
+          0.2,
+          0.61806
+        ],
+        [
+          0.1,
+          0.5,
+          0.59915
+        ],
+        [
+          0.1,
+          0.8,
+          0.5
+        ],
+        [
+          0.1,
+          0.9,
+          0.5
+        ],
+        [
+          0.1,
+          0.95,
+          0.5
+        ],
+        [
+          0.1,
+          1.0,
+          0.5
+        ],
+        [
+          1.0,
+          0.0,
+          0.62202
+        ],
+        [
+          1.0,
+          0.05,
+          0.58296
+        ],
+        [
+          1.0,
+          0.1,
+          0.5
+        ],
+        [
+          1.0,
+          0.2,
+          0.5
+        ],
+        [
+          1.0,
+          0.5,
+          0.5
+        ],
+        [
+          1.0,
+          0.8,
+          0.5
+        ],
+        [
+          1.0,
+          0.9,
+          0.5
+        ],
+        [
+          1.0,
+          0.95,
+          0.5
+        ],
+        [
+          1.0,
+          1.0,
+          0.5
+        ],
+        [
+          10.0,
+          0.0,
+          0.61993
+        ],
+        [
+          10.0,
+          0.05,
+          0.5
+        ],
+        [
+          10.0,
+          0.1,
+          0.5
+        ],
+        [
+          10.0,
+          0.2,
+          0.5
+        ],
+        [
+          10.0,
+          0.5,
+          0.5
+        ],
+        [
+          10.0,
+          0.8,
+          0.5
+        ],
+        [
+          10.0,
+          0.9,
+          0.5
+        ],
+        [
+          10.0,
+          0.95,
+          0.5
+        ],
+        [
+          10.0,
+          1.0,
+          0.5
+        ]
+      ]
+    }
+  },
   "model": {
     "class": "SGDClassifier",
     "module": "sklearn.linear_model.stochastic_gradient",