From eb1b67045a203314be21fbfed7b04e7cc0d710ba Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Wed, 21 Sep 2016 11:35:21 -0400 Subject: [PATCH] Save grid_search performance metrics --- cognoml/analysis.py | 7 +- data/api/hippo-output.json | 1263 ++++++++++++++++++++++++++++++++++++ 2 files changed, 1269 insertions(+), 1 deletion(-) diff --git a/cognoml/analysis.py b/cognoml/analysis.py index 4906e39..f51904c 100644 --- a/cognoml/analysis.py +++ b/cognoml/analysis.py @@ -54,7 +54,6 @@ def classify(sample_id, mutation_status, **kwargs): obs_df['testing'] = obs_df.sample_id.isin(X_test.index).astype(int) pipeline.fit(X=X_train, y=y_train) - #cv_score_df = grid_scores_to_df(clf_grid.grid_scores_) predict_df = pd.DataFrame.from_items([ ('sample_id', X_whole.index), @@ -95,6 +94,12 @@ def classify(sample_id, mutation_status, **kwargs): performance['cv'] = {'auroc': round(clf_grid.best_score_, 5)} results['performance'] = performance + gs = collections.OrderedDict() + gs['mean_scores'] = utils.mean_grid_scores_to_df(clf_grid.grid_scores_) + gs['fold_scores'] = utils.grid_scores_to_df(clf_grid.grid_scores_) + gs = utils.value_map(gs, utils.df_to_datatables) + results['grid_search'] = gs + results['model'] = utils.model_info(clf_grid.best_estimator_) feature_df = utils.get_feature_df(clf_grid.best_estimator_, X.columns) diff --git a/data/api/hippo-output.json b/data/api/hippo-output.json index 48dddce..8d4f4bf 100644 --- a/data/api/hippo-output.json +++ b/data/api/hippo-output.json @@ -32,6 +32,1269 @@ "auroc": 0.62524 } }, + "grid_search": { + "mean_scores": { + "columns": [ + "alpha", + "l1_ratio", + "score" + ], + "data": [ + [ + 0.0001, + 0.0, + 0.55805 + ], + [ + 0.0001, + 0.05, + 0.56247 + ], + [ + 0.0001, + 0.1, + 0.56111 + ], + [ + 0.0001, + 0.2, + 0.56368 + ], + [ + 0.0001, + 0.5, + 0.56555 + ], + [ + 0.0001, + 0.8, + 0.55769 + ], + [ + 0.0001, + 0.9, + 0.55852 + ], + [ + 0.0001, + 0.95, + 0.56522 + ], + [ + 0.0001, + 1.0, + 0.5652 + ], + [ + 0.001, + 0.0, + 0.58292 + ], + [ + 0.001, + 0.05, + 0.59085 + ], + [ + 0.001, + 0.1, + 0.60311 + ], + [ + 0.001, + 0.2, + 0.61103 + ], + [ + 0.001, + 0.5, + 0.61417 + ], + [ + 0.001, + 0.8, + 0.61382 + ], + [ + 0.001, + 0.9, + 0.61385 + ], + [ + 0.001, + 0.95, + 0.61387 + ], + [ + 0.001, + 1.0, + 0.61392 + ], + [ + 0.01, + 0.0, + 0.62038 + ], + [ + 0.01, + 0.05, + 0.62385 + ], + [ + 0.01, + 0.1, + 0.62389 + ], + [ + 0.01, + 0.2, + 0.62418 + ], + [ + 0.01, + 0.5, + 0.62415 + ], + [ + 0.01, + 0.8, + 0.62484 + ], + [ + 0.01, + 0.9, + 0.62461 + ], + [ + 0.01, + 0.95, + 0.6245 + ], + [ + 0.01, + 1.0, + 0.62333 + ], + [ + 0.1, + 0.0, + 0.62524 + ], + [ + 0.1, + 0.05, + 0.62466 + ], + [ + 0.1, + 0.1, + 0.62411 + ], + [ + 0.1, + 0.2, + 0.61806 + ], + [ + 0.1, + 0.5, + 0.59915 + ], + [ + 0.1, + 0.8, + 0.5 + ], + [ + 0.1, + 0.9, + 0.5 + ], + [ + 0.1, + 0.95, + 0.5 + ], + [ + 0.1, + 1.0, + 0.5 + ], + [ + 1.0, + 0.0, + 0.62202 + ], + [ + 1.0, + 0.05, + 0.58296 + ], + [ + 1.0, + 0.1, + 0.5 + ], + [ + 1.0, + 0.2, + 0.5 + ], + [ + 1.0, + 0.5, + 0.5 + ], + [ + 1.0, + 0.8, + 0.5 + ], + [ + 1.0, + 0.9, + 0.5 + ], + [ + 1.0, + 0.95, + 0.5 + ], + [ + 1.0, + 1.0, + 0.5 + ], + [ + 10.0, + 0.0, + 0.61993 + ], + [ + 10.0, + 0.05, + 0.5 + ], + [ + 10.0, + 0.1, + 0.5 + ], + [ + 10.0, + 0.2, + 0.5 + ], + [ + 10.0, + 0.5, + 0.5 + ], + [ + 10.0, + 0.8, + 0.5 + ], + [ + 10.0, + 0.9, + 0.5 + ], + [ + 10.0, + 0.95, + 0.5 + ], + [ + 10.0, + 1.0, + 0.5 + ] + ] + }, + "fold_scores": { + "columns": [ + "alpha", + "fold", + "l1_ratio", + "score" + ], + "data": [ + [ + 0.0001, + 0, + 0.0, + 0.48733 + ], + [ + 0.0001, + 1, + 0.0, + 0.59161 + ], + [ + 0.0001, + 2, + 0.0, + 0.59526 + ], + [ + 0.0001, + 0, + 0.05, + 0.49873 + ], + [ + 0.0001, + 1, + 0.05, + 0.58957 + ], + [ + 0.0001, + 2, + 0.05, + 0.59915 + ], + [ + 0.0001, + 0, + 0.1, + 0.50386 + ], + [ + 0.0001, + 1, + 0.1, + 0.58154 + ], + [ + 0.0001, + 2, + 0.1, + 0.59797 + ], + [ + 0.0001, + 0, + 0.2, + 0.50539 + ], + [ + 0.0001, + 1, + 0.2, + 0.5915 + ], + [ + 0.0001, + 2, + 0.2, + 0.5942 + ], + [ + 0.0001, + 0, + 0.5, + 0.5232 + ], + [ + 0.0001, + 1, + 0.5, + 0.59996 + ], + [ + 0.0001, + 2, + 0.5, + 0.57353 + ], + [ + 0.0001, + 0, + 0.8, + 0.50881 + ], + [ + 0.0001, + 1, + 0.8, + 0.5889 + ], + [ + 0.0001, + 2, + 0.8, + 0.57541 + ], + [ + 0.0001, + 0, + 0.9, + 0.51105 + ], + [ + 0.0001, + 1, + 0.9, + 0.57999 + ], + [ + 0.0001, + 2, + 0.9, + 0.58455 + ], + [ + 0.0001, + 0, + 0.95, + 0.52634 + ], + [ + 0.0001, + 1, + 0.95, + 0.5814 + ], + [ + 0.0001, + 2, + 0.95, + 0.58795 + ], + [ + 0.0001, + 0, + 1.0, + 0.52799 + ], + [ + 0.0001, + 1, + 1.0, + 0.58302 + ], + [ + 0.0001, + 2, + 1.0, + 0.58462 + ], + [ + 0.001, + 0, + 0.0, + 0.53017 + ], + [ + 0.001, + 1, + 0.0, + 0.59751 + ], + [ + 0.001, + 2, + 0.0, + 0.62112 + ], + [ + 0.001, + 0, + 0.05, + 0.54779 + ], + [ + 0.001, + 1, + 0.05, + 0.59732 + ], + [ + 0.001, + 2, + 0.05, + 0.62748 + ], + [ + 0.001, + 0, + 0.1, + 0.57378 + ], + [ + 0.001, + 1, + 0.1, + 0.60776 + ], + [ + 0.001, + 2, + 0.1, + 0.62781 + ], + [ + 0.001, + 0, + 0.2, + 0.58603 + ], + [ + 0.001, + 1, + 0.2, + 0.6182 + ], + [ + 0.001, + 2, + 0.2, + 0.62887 + ], + [ + 0.001, + 0, + 0.5, + 0.59417 + ], + [ + 0.001, + 1, + 0.5, + 0.62006 + ], + [ + 0.001, + 2, + 0.5, + 0.6283 + ], + [ + 0.001, + 0, + 0.8, + 0.59471 + ], + [ + 0.001, + 1, + 0.8, + 0.61823 + ], + [ + 0.001, + 2, + 0.8, + 0.62853 + ], + [ + 0.001, + 0, + 0.9, + 0.59497 + ], + [ + 0.001, + 1, + 0.9, + 0.61808 + ], + [ + 0.001, + 2, + 0.9, + 0.62852 + ], + [ + 0.001, + 0, + 0.95, + 0.59507 + ], + [ + 0.001, + 1, + 0.95, + 0.61803 + ], + [ + 0.001, + 2, + 0.95, + 0.62852 + ], + [ + 0.001, + 0, + 1.0, + 0.59516 + ], + [ + 0.001, + 1, + 1.0, + 0.61811 + ], + [ + 0.001, + 2, + 1.0, + 0.62852 + ], + [ + 0.01, + 0, + 0.0, + 0.60689 + ], + [ + 0.01, + 1, + 0.0, + 0.62088 + ], + [ + 0.01, + 2, + 0.0, + 0.63339 + ], + [ + 0.01, + 0, + 0.05, + 0.6124 + ], + [ + 0.01, + 1, + 0.05, + 0.62786 + ], + [ + 0.01, + 2, + 0.05, + 0.63128 + ], + [ + 0.01, + 0, + 0.1, + 0.61214 + ], + [ + 0.01, + 1, + 0.1, + 0.62839 + ], + [ + 0.01, + 2, + 0.1, + 0.63114 + ], + [ + 0.01, + 0, + 0.2, + 0.61218 + ], + [ + 0.01, + 1, + 0.2, + 0.62935 + ], + [ + 0.01, + 2, + 0.2, + 0.63102 + ], + [ + 0.01, + 0, + 0.5, + 0.61277 + ], + [ + 0.01, + 1, + 0.5, + 0.62914 + ], + [ + 0.01, + 2, + 0.5, + 0.63054 + ], + [ + 0.01, + 0, + 0.8, + 0.61221 + ], + [ + 0.01, + 1, + 0.8, + 0.63205 + ], + [ + 0.01, + 2, + 0.8, + 0.63027 + ], + [ + 0.01, + 0, + 0.9, + 0.61144 + ], + [ + 0.01, + 1, + 0.9, + 0.63253 + ], + [ + 0.01, + 2, + 0.9, + 0.62988 + ], + [ + 0.01, + 0, + 0.95, + 0.61116 + ], + [ + 0.01, + 1, + 0.95, + 0.6326 + ], + [ + 0.01, + 2, + 0.95, + 0.62974 + ], + [ + 0.01, + 0, + 1.0, + 0.61098 + ], + [ + 0.01, + 1, + 1.0, + 0.62954 + ], + [ + 0.01, + 2, + 1.0, + 0.6295 + ], + [ + 0.1, + 0, + 0.0, + 0.61255 + ], + [ + 0.1, + 1, + 0.0, + 0.62922 + ], + [ + 0.1, + 2, + 0.0, + 0.63395 + ], + [ + 0.1, + 0, + 0.05, + 0.61279 + ], + [ + 0.1, + 1, + 0.05, + 0.62893 + ], + [ + 0.1, + 2, + 0.05, + 0.63227 + ], + [ + 0.1, + 0, + 0.1, + 0.61169 + ], + [ + 0.1, + 1, + 0.1, + 0.62912 + ], + [ + 0.1, + 2, + 0.1, + 0.63152 + ], + [ + 0.1, + 0, + 0.2, + 0.60237 + ], + [ + 0.1, + 1, + 0.2, + 0.62988 + ], + [ + 0.1, + 2, + 0.2, + 0.62192 + ], + [ + 0.1, + 0, + 0.5, + 0.57488 + ], + [ + 0.1, + 1, + 0.5, + 0.61841 + ], + [ + 0.1, + 2, + 0.5, + 0.60419 + ], + [ + 0.1, + 0, + 0.8, + 0.5 + ], + [ + 0.1, + 1, + 0.8, + 0.5 + ], + [ + 0.1, + 2, + 0.8, + 0.5 + ], + [ + 0.1, + 0, + 0.9, + 0.5 + ], + [ + 0.1, + 1, + 0.9, + 0.5 + ], + [ + 0.1, + 2, + 0.9, + 0.5 + ], + [ + 0.1, + 0, + 0.95, + 0.5 + ], + [ + 0.1, + 1, + 0.95, + 0.5 + ], + [ + 0.1, + 2, + 0.95, + 0.5 + ], + [ + 0.1, + 0, + 1.0, + 0.5 + ], + [ + 0.1, + 1, + 1.0, + 0.5 + ], + [ + 0.1, + 2, + 1.0, + 0.5 + ], + [ + 1.0, + 0, + 0.0, + 0.60965 + ], + [ + 1.0, + 1, + 0.0, + 0.62621 + ], + [ + 1.0, + 2, + 0.0, + 0.6302 + ], + [ + 1.0, + 0, + 0.05, + 0.57488 + ], + [ + 1.0, + 1, + 0.05, + 0.56982 + ], + [ + 1.0, + 2, + 0.05, + 0.60419 + ], + [ + 1.0, + 0, + 0.1, + 0.5 + ], + [ + 1.0, + 1, + 0.1, + 0.5 + ], + [ + 1.0, + 2, + 0.1, + 0.5 + ], + [ + 1.0, + 0, + 0.2, + 0.5 + ], + [ + 1.0, + 1, + 0.2, + 0.5 + ], + [ + 1.0, + 2, + 0.2, + 0.5 + ], + [ + 1.0, + 0, + 0.5, + 0.5 + ], + [ + 1.0, + 1, + 0.5, + 0.5 + ], + [ + 1.0, + 2, + 0.5, + 0.5 + ], + [ + 1.0, + 0, + 0.8, + 0.5 + ], + [ + 1.0, + 1, + 0.8, + 0.5 + ], + [ + 1.0, + 2, + 0.8, + 0.5 + ], + [ + 1.0, + 0, + 0.9, + 0.5 + ], + [ + 1.0, + 1, + 0.9, + 0.5 + ], + [ + 1.0, + 2, + 0.9, + 0.5 + ], + [ + 1.0, + 0, + 0.95, + 0.5 + ], + [ + 1.0, + 1, + 0.95, + 0.5 + ], + [ + 1.0, + 2, + 0.95, + 0.5 + ], + [ + 1.0, + 0, + 1.0, + 0.5 + ], + [ + 1.0, + 1, + 1.0, + 0.5 + ], + [ + 1.0, + 2, + 1.0, + 0.5 + ], + [ + 10.0, + 0, + 0.0, + 0.6079 + ], + [ + 10.0, + 1, + 0.0, + 0.62431 + ], + [ + 10.0, + 2, + 0.0, + 0.62759 + ], + [ + 10.0, + 0, + 0.05, + 0.5 + ], + [ + 10.0, + 1, + 0.05, + 0.5 + ], + [ + 10.0, + 2, + 0.05, + 0.5 + ], + [ + 10.0, + 0, + 0.1, + 0.5 + ], + [ + 10.0, + 1, + 0.1, + 0.5 + ], + [ + 10.0, + 2, + 0.1, + 0.5 + ], + [ + 10.0, + 0, + 0.2, + 0.5 + ], + [ + 10.0, + 1, + 0.2, + 0.5 + ], + [ + 10.0, + 2, + 0.2, + 0.5 + ], + [ + 10.0, + 0, + 0.5, + 0.5 + ], + [ + 10.0, + 1, + 0.5, + 0.5 + ], + [ + 10.0, + 2, + 0.5, + 0.5 + ], + [ + 10.0, + 0, + 0.8, + 0.5 + ], + [ + 10.0, + 1, + 0.8, + 0.5 + ], + [ + 10.0, + 2, + 0.8, + 0.5 + ], + [ + 10.0, + 0, + 0.9, + 0.5 + ], + [ + 10.0, + 1, + 0.9, + 0.5 + ], + [ + 10.0, + 2, + 0.9, + 0.5 + ], + [ + 10.0, + 0, + 0.95, + 0.5 + ], + [ + 10.0, + 1, + 0.95, + 0.5 + ], + [ + 10.0, + 2, + 0.95, + 0.5 + ], + [ + 10.0, + 0, + 1.0, + 0.5 + ], + [ + 10.0, + 1, + 1.0, + 0.5 + ], + [ + 10.0, + 2, + 1.0, + 0.5 + ] + ] + } + }, "model": { "class": "SGDClassifier", "module": "sklearn.linear_model.stochastic_gradient",