diff --git a/.DS_Store b/.DS_Store index 7a51f9c..4481711 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/dividend_policy_predictor.py b/dividend_policy_predictor.py index 5d68f04..cfb1bbf 100644 --- a/dividend_policy_predictor.py +++ b/dividend_policy_predictor.py @@ -166,9 +166,6 @@ def rank_columns_by_correlation(df, threshold=0.9): plt.xticks(rotation=45, ha='right', rotation_mode="anchor") plt.show() -# ============================================================================ - - # Now let's remove the features one by one from the least important one X_train_temp = X_train_oversampled.copy() X_validate_temp = X_validate_transformed.copy() @@ -221,6 +218,7 @@ def rank_columns_by_correlation(df, threshold=0.9): with open('result_df.pkl', 'rb') as file: result_df = pickle.load(file) +# ============================================================================ # Model Selection X_train_oversampled.info() @@ -242,7 +240,7 @@ def rank_columns_by_correlation(df, threshold=0.9): # Create and save model best_model_lr = LogisticRegression(**best_params_lr, solver='liblinear', n_jobs=-1) -with open('best_models_lr.pkl', 'wb') as file: +with open('storage_files/best_models_lr.pkl', 'wb') as file: pickle.dump(best_model_lr, file) @@ -278,7 +276,7 @@ def objective_function(trial): # Create and save model best_model_lr = LogisticRegression(**best_params_lr, solver='liblinear', n_jobs=-1) -with open('best_models_lr.pkl', 'wb') as file: +with open('storage_files/best_models_lr.pkl', 'wb') as file: pickle.dump(best_model_lr, file) # Decision Tree @@ -312,7 +310,7 @@ def objective_function(trial): # Create and save model best_model_dt = DecisionTreeClassifier(**best_params_dt) -with open('best_models_dt.pkl', 'wb') as file: +with open('storage_files/best_models_dt.pkl', 'wb') as file: pickle.dump(best_model_dt, file) # KNN @@ -347,7 +345,7 @@ def objective_function(trial): # Create and save model best_model_knn = KNeighborsClassifier(**best_params_knn) -with open('best_models_knn.pkl', 'wb') as file: +with open('storage_files/best_models_knn.pkl', 'wb') as file: pickle.dump(best_model_knn, file) # Random Forest @@ -383,7 +381,7 @@ def objective_function(trial): # Create and save model best_model_rf = RandomForestClassifier(**best_params_rf, n_jobs=-1) -with open('best_models_rf.pkl', 'wb') as file: +with open('storage_files/best_models_rf.pkl', 'wb') as file: pickle.dump(best_model_rf, file) # XgBoost @@ -443,7 +441,7 @@ def objective_function(trial): print("Best ROC-AUC Score: ", study_xgb.best_value) best_model_xgb = XGBClassifier(**best_params_xgb, use_label_encoder=False, n_jobs=-1) -with open('best_models_xgb.pkl', 'wb') as file: +with open('storage_files/best_models_xgb.pkl', 'wb') as file: pickle.dump(best_model_xgb, file) # Label encode categorical features with many categories @@ -457,15 +455,15 @@ def objective_function(trial): # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) # Model selection - Compare Performance -with open('best_models_lr.pkl', 'rb') as file: +with open('storage_files/best_models_lr.pkl', 'rb') as file: best_model_lr = pickle.load(file) -with open('best_models_dt.pkl', 'rb') as file: +with open('storage_files/best_models_dt.pkl', 'rb') as file: best_model_dt = pickle.load(file) -with open('best_models_knn.pkl', 'rb') as file: +with open('storage_files/best_models_knn.pkl', 'rb') as file: best_model_knn = pickle.load(file) -with open('best_models_rf.pkl', 'rb') as file: +with open('storage_files/best_models_rf.pkl', 'rb') as file: best_model_rf = pickle.load(file) -with open('best_models_xgb.pkl', 'rb') as file: +with open('storage_files/best_models_xgb.pkl', 'rb') as file: best_model_xgb = pickle.load(file) print("Testing Performances...Please wait") diff --git a/feature_engineering.py b/feature_engineering.py index 0115b5e..1590763 100644 --- a/feature_engineering.py +++ b/feature_engineering.py @@ -8,17 +8,6 @@ from sklearn.ensemble import RandomForestClassifier # from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, make_scorer import sklearn.metrics as skm -from sklearn.neighbors import KNeighborsClassifier -from sklearn.linear_model import LogisticRegression -from sklearn.tree import DecisionTreeClassifier -from sklearn.svm import SVC -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OrdinalEncoder, LabelEncoder -from sklearn.compose import ColumnTransformer -from sklearn.model_selection import cross_val_score, GridSearchCV -import pickle -import optuna -from xgboost import XGBClassifier warnings.filterwarnings('ignore') @@ -36,7 +25,8 @@ X_validate_temp = X_validate_transformed.copy() # Initialize the result dataframe -result_df = pd.DataFrame(columns=['Features_Removed', 'ROC_Score']) +result_df = pd.DataFrame(columns=['Features_Removed', 'ROC_Score', 'f1', + 'accuracy', 'precision', 'recall']) # First, evaluate performance using all features randomForestModel = RandomForestClassifier(max_features=None) @@ -44,16 +34,24 @@ randomForestModel.fit(X_train_temp, y_train_oversampled) # Predict probabilities on test data y_pred_probs = randomForestModel.predict_proba(X_validate_temp)[:, 1] +y_pred = randomForestModel.predict(X_validate_temp) # Different metrics of classification roc_score = skm.roc_auc_score(y_test, y_pred_probs) -# f1_score = skm.f1_score(y_test, y_pred_probs, average=None) - - +f1_score = skm.f1_score(y_test, y_pred, average=None)[0] +accuracy = skm.accuracy_score(y_test, y_pred) +precision = skm.precision_score(y_test, y_pred, average=None)[0] +recall = skm.recall_score(y_test, y_pred, average=None)[0] # Append the result to the result dataframe -roc_dict = {'Features_Removed': 'None', 'no_features_used': len(X_train_temp.columns), 'ROC_Score': roc_score} +roc_dict = { + 'Features_Removed': 'None', + 'no_features_used': len(X_train_temp.columns), + 'ROC_Score': roc_score, 'f1': f1_score, 'accuracy': accuracy, + 'precision': precision, 'recall': recall} result_df = pd.DataFrame([roc_dict]) total_num = len(X_train_temp.columns) -print(f"Feature_Removed: None, Number of features used: {len(X_train_temp.columns)}, ROC_AUC_Score: {roc_score}") +print(f"Feature_Removed: None, Number of features used: {len(X_train_temp.columns)}, \ +ROC_AUC_Score: {roc_score}, F1: {f1_score}, Accuracy: {accuracy}, Precision: {precision} \ + Recall: {recall}") result_df.to_csv("storage_files/result_df.csv",index=False) # Sort importance_table by Importance in ascending order to start with the least important @@ -71,24 +69,36 @@ # Compute ROC score roc_score = skm.roc_auc_score(y_test, y_pred_probs) # Append the result to the result dataframe - roc_dict = {'Features_Removed': row['Feature'], 'no_features_used': len(X_train_temp.columns), 'ROC_Score': roc_score} + roc_dict = { + 'Features_Removed': row['Feature'], + 'no_features_used': len(X_train_temp.columns), + 'ROC_Score': roc_score, 'f1': f1_score, 'accuracy': accuracy, + 'precision': precision, 'recall': recall} pd.DataFrame([roc_dict]).to_csv("storage_files/result_df.csv",mode='a',index=False,header=False) result_df = pd.concat([result_df, pd.DataFrame([roc_dict])]) - print( - f"Feature_Removed: {row['Feature']}, Number of features used: {len(X_train_temp.columns)}, ROC_AUC_Score: {roc_score}") + print(f"Feature_Removed: {row['Feature']}, Number of features used: {len(X_train_temp.columns)}, \ + ROC_AUC_Score: {roc_score}, F1: {f1_score}, Accuracy: {accuracy}, Precision: {precision} \ + Recall: {recall}") # If only one feature left, break the loop if X_train_temp.shape[1] == 1: break -if __name__ == '__main__': - # Plot a bar chart to visualize ROC scores +def plot(metric: str): + """ + metric: 'ROC_Score', 'f1', 'accuracy', 'precision' or 'recall'. + """ plt.figure(figsize=(20, 10)) - sns.barplot(data=result_df, x="no_features_used", y="ROC_Score") - plt.title("ROC scores") + sns.barplot(data=result_df, x="no_features_used", y=metric) + plt.title(metric) plt.subplots_adjust(bottom=0.2, top=0.95) plt.xticks(rotation=45, ha='right', rotation_mode="anchor") plt.show() +if __name__ == '__main__': + for metric in ['ROC_Score', 'f1', 'accuracy', 'precision', 'recall']: + plot(metric) + + # Find out the one with max roc_score. max_roc_score = result_df.iloc[0]['ROC_Score'] max_inds = [0] @@ -100,15 +110,8 @@ max_inds.append(index) max_no_features = [total_num-i for i in max_inds] print(f'Conclusion: The best model is to use {max_no_features} features.') - -# Save the results -# with open('storage_files/result_df.pkl', 'wb') as file: -# pickle.dump(result_df, file) -# with open('storage_files/importance_table_sorted.pkl', 'wb') as file: -# pickle.dump(importance_table_sorted, file) - -# Load the results -# with open('storage_files/result_df.pkl', 'rb') as file: -# result_df = pickle.load(file) - - +f = open('storage_files/max_no_features.txt','w') +for i in max_no_features: + f.write(str(i)) + f.write('\n') +f.close() diff --git a/importance_table_sorted.pkl b/importance_table_sorted.pkl deleted file mode 100644 index 23d0124..0000000 Binary files a/importance_table_sorted.pkl and /dev/null differ diff --git a/model_training_top_36_features.ipynb b/model_training_top_36_features.ipynb new file mode 100644 index 0000000..2d2b230 --- /dev/null +++ b/model_training_top_36_features.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyM9xRgmNnsWQefamDumf5mX"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","source":["pip install optuna"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PI5SGFmpCjhi","executionInfo":{"status":"ok","timestamp":1715571689335,"user_tz":-480,"elapsed":12218,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"abfdaa80-5d65-471d-89ad-b1003c372133"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting optuna\n"," Downloading optuna-3.6.1-py3-none-any.whl (380 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m380.1/380.1 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting alembic>=1.5.0 (from optuna)\n"," Downloading alembic-1.13.1-py3-none-any.whl (233 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.4/233.4 kB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting colorlog (from optuna)\n"," Downloading colorlog-6.8.2-py3-none-any.whl (11 kB)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from optuna) (1.25.2)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from optuna) (24.0)\n","Requirement already satisfied: sqlalchemy>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from optuna) (2.0.30)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from optuna) (4.66.4)\n","Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from optuna) (6.0.1)\n","Collecting Mako (from alembic>=1.5.0->optuna)\n"," Downloading Mako-1.3.3-py3-none-any.whl (78 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.8/78.8 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: typing-extensions>=4 in /usr/local/lib/python3.10/dist-packages (from alembic>=1.5.0->optuna) (4.11.0)\n","Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy>=1.3.0->optuna) (3.0.3)\n","Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from Mako->alembic>=1.5.0->optuna) (2.1.5)\n","Installing collected packages: Mako, colorlog, alembic, optuna\n","Successfully installed Mako-1.3.3 alembic-1.13.1 colorlog-6.8.2 optuna-3.6.1\n"]}]},{"cell_type":"code","execution_count":3,"metadata":{"id":"WMlZHH_fCW4U","executionInfo":{"status":"ok","timestamp":1715571698655,"user_tz":-480,"elapsed":397,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}}},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","import warnings\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","from imblearn.over_sampling import SMOTENC\n","from sklearn.model_selection import train_test_split\n","from sklearn.ensemble import RandomForestClassifier\n","from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, make_scorer\n","# import sklearn.metrics as skm\n","from sklearn.neighbors import KNeighborsClassifier\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.svm import SVC\n","from sklearn.pipeline import Pipeline\n","from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n","from sklearn.compose import ColumnTransformer\n","from sklearn.model_selection import cross_val_score, GridSearchCV\n","import pickle\n","import optuna\n","from xgboost import XGBClassifier\n","\n","warnings.filterwarnings('ignore')"]},{"cell_type":"code","source":["# Import the files from data_preprocessing.\n","X_train_oversampled = pd.read_csv(\"storage_files/X_train_oversampled.csv\")\n","X_validate_transformed = pd.read_csv(\"storage_files/X_validate_transformed.csv\")\n","X_test_transformed = pd.read_csv(\"storage_files/X_test_transformed.csv\")\n","y_train_oversampled = pd.read_csv(\"storage_files/y_train_oversampled.csv\")\n","y_validate = pd.read_csv(\"storage_files/y_validate.csv\")\n","y_test = pd.read_csv(\"storage_files/y_test.csv\")\n","importance_table = pd.read_csv(\"storage_files/importance_table.csv\")"],"metadata":{"id":"xQHcyZraDXox","executionInfo":{"status":"ok","timestamp":1715571892066,"user_tz":-480,"elapsed":510,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["f = open('storage_files/max_no_features.txt','r')\n","max_no_features = int(f.readline())\n","f.close()\n","max_features = list(importance_table[:max_no_features]['Feature'])\n","X_train_oversampled = X_train_oversampled[max_features]\n","X_validate_transformed = X_validate_transformed[max_features]\n","X_test_transformed = X_test_transformed[max_features]"],"metadata":{"id":"DULv2CQqDbCO","executionInfo":{"status":"ok","timestamp":1715571904768,"user_tz":-480,"elapsed":412,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["# Logistic Regression\n","print('\\nLogistic Regression started.')\n","def objective_function(trial):\n"," C = trial.suggest_float('C', 0.1, 10, log=True)\n"," penalty = trial.suggest_categorical('penalty', ['l1', 'l2'])\n","\n"," model = LogisticRegression(\n"," C=C,\n"," penalty=penalty,\n"," solver='liblinear',\n"," n_jobs=-1\n"," )\n"," metrics = ['roc_auc','accuracy','precison','recall','f1-score']\n"," # Using cross_val_score to get the average precision score for each fold\n"," scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc')\n"," roc_auc = np.mean(scores)\n"," # Printing intermediate results\n"," print(f\"Trial {trial.number}, C: {C}, penalty: {penalty}, ROC-AUC: {roc_auc}\")\n"," return roc_auc\n","\n","\n","study_lr = optuna.create_study(direction=\"maximize\")\n","study_lr.optimize(objective_function, n_trials=100)\n","\n","best_params_lr = study_lr.best_params\n","print(\"Best Parameters: \", best_params_lr)\n","print(\"Best ROC-AUC Score: \", study_lr.best_value)\n","\n","# Create and save model\n","best_model_lr = LogisticRegression(**best_params_lr, solver='liblinear', n_jobs=-1)\n","with open('storage_files/best_models_lr_36_features.pkl', 'wb') as file:\n"," pickle.dump(best_model_lr, file)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AblFL7uKDgge","executionInfo":{"status":"ok","timestamp":1715572647591,"user_tz":-480,"elapsed":724940,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"35612b6e-362f-4606-e3f7-97a74c99dfe7"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:22,557] A new study created in memory with name: no-name-9aaf1113-3d34-435d-9f20-74f02769ce02\n"]},{"output_type":"stream","name":"stdout","text":["\n","Logistic Regression started.\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:24,914] Trial 0 finished with value: 0.7025894721285637 and parameters: {'C': 0.29358461003628333, 'penalty': 'l1'}. Best is trial 0 with value: 0.7025894721285637.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 0, C: 0.29358461003628333, penalty: l1, ROC-AUC: 0.7025894721285637\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:30,408] Trial 1 finished with value: 0.7125044730681401 and parameters: {'C': 1.398129321890534, 'penalty': 'l1'}. Best is trial 1 with value: 0.7125044730681401.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 1, C: 1.398129321890534, penalty: l1, ROC-AUC: 0.7125044730681401\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:37,629] Trial 2 finished with value: 0.7133279107672936 and parameters: {'C': 2.0660386381374964, 'penalty': 'l1'}. Best is trial 2 with value: 0.7133279107672936.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 2, C: 2.0660386381374964, penalty: l1, ROC-AUC: 0.7133279107672936\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:43,012] Trial 3 finished with value: 0.5864743834205929 and parameters: {'C': 2.5397468124693234, 'penalty': 'l2'}. Best is trial 2 with value: 0.7133279107672936.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 3, C: 2.5397468124693234, penalty: l2, ROC-AUC: 0.5864743834205929\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:55,409] Trial 4 finished with value: 0.7135673803437385 and parameters: {'C': 2.3623902869591005, 'penalty': 'l1'}. Best is trial 4 with value: 0.7135673803437385.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 4, C: 2.3623902869591005, penalty: l1, ROC-AUC: 0.7135673803437385\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:45:59,633] Trial 5 finished with value: 0.5760005561043279 and parameters: {'C': 4.812381713703083, 'penalty': 'l2'}. Best is trial 4 with value: 0.7135673803437385.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 5, C: 4.812381713703083, penalty: l2, ROC-AUC: 0.5760005561043279\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:03,928] Trial 6 finished with value: 0.5809454743548386 and parameters: {'C': 0.19812841789359492, 'penalty': 'l2'}. Best is trial 4 with value: 0.7135673803437385.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 6, C: 0.19812841789359492, penalty: l2, ROC-AUC: 0.5809454743548386\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:07,006] Trial 7 finished with value: 0.691808087602728 and parameters: {'C': 0.15733622241432405, 'penalty': 'l1'}. Best is trial 4 with value: 0.7135673803437385.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 7, C: 0.15733622241432405, penalty: l1, ROC-AUC: 0.691808087602728\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:15,494] Trial 8 finished with value: 0.7147456063346245 and parameters: {'C': 9.622308611571249, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 8, C: 9.622308611571249, penalty: l1, ROC-AUC: 0.7147456063346245\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:18,944] Trial 9 finished with value: 0.7107017653905947 and parameters: {'C': 0.8280334886565537, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 9, C: 0.8280334886565537, penalty: l1, ROC-AUC: 0.7107017653905947\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:24,830] Trial 10 finished with value: 0.6062432225840503 and parameters: {'C': 9.477524528035168, 'penalty': 'l2'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 10, C: 9.477524528035168, penalty: l2, ROC-AUC: 0.6062432225840503\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:38,209] Trial 11 finished with value: 0.7146684606857419 and parameters: {'C': 8.442569940260936, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 11, C: 8.442569940260936, penalty: l1, ROC-AUC: 0.7146684606857419\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:43,898] Trial 12 finished with value: 0.7146913717027589 and parameters: {'C': 8.492987175677744, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 12, C: 8.492987175677744, penalty: l1, ROC-AUC: 0.7146913717027589\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:57,015] Trial 13 finished with value: 0.7144079862652533 and parameters: {'C': 4.957637399942353, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 13, C: 4.957637399942353, penalty: l1, ROC-AUC: 0.7144079862652533\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:46:59,162] Trial 14 finished with value: 0.7070023401386875 and parameters: {'C': 0.45621059158646077, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 14, C: 0.45621059158646077, penalty: l1, ROC-AUC: 0.7070023401386875\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:08,135] Trial 15 finished with value: 0.7144183071469918 and parameters: {'C': 5.07696091195844, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 15, C: 5.07696091195844, penalty: l1, ROC-AUC: 0.7144183071469918\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:20,179] Trial 16 finished with value: 0.7142830891890946 and parameters: {'C': 4.3421690541158195, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 16, C: 4.3421690541158195, penalty: l1, ROC-AUC: 0.7142830891890946\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:23,090] Trial 17 finished with value: 0.709747694681161 and parameters: {'C': 0.6825465261316978, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 17, C: 0.6825465261316978, penalty: l1, ROC-AUC: 0.709747694681161\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:27,414] Trial 18 finished with value: 0.5502662050772764 and parameters: {'C': 9.49692522147848, 'penalty': 'l2'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 18, C: 9.49692522147848, penalty: l2, ROC-AUC: 0.5502662050772764\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:37,025] Trial 19 finished with value: 0.7140203365435986 and parameters: {'C': 3.275027927105184, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 19, C: 3.275027927105184, penalty: l1, ROC-AUC: 0.7140203365435986\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:42,132] Trial 20 finished with value: 0.7124334390995466 and parameters: {'C': 1.3543768403038026, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 20, C: 1.3543768403038026, penalty: l1, ROC-AUC: 0.7124334390995466\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:46,992] Trial 21 finished with value: 0.7146753307189 and parameters: {'C': 8.051392864360068, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 21, C: 8.051392864360068, penalty: l1, ROC-AUC: 0.7146753307189\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:47:57,716] Trial 22 finished with value: 0.7145596040657086 and parameters: {'C': 6.402945837464176, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 22, C: 6.402945837464176, penalty: l1, ROC-AUC: 0.7145596040657086\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:06,485] Trial 23 finished with value: 0.7146069707753497 and parameters: {'C': 6.953933251535305, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 23, C: 6.953933251535305, penalty: l1, ROC-AUC: 0.7146069707753497\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:15,593] Trial 24 finished with value: 0.7141177281559227 and parameters: {'C': 3.5534806762484763, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 24, C: 3.5534806762484763, penalty: l1, ROC-AUC: 0.7141177281559227\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:18,546] Trial 25 finished with value: 0.714705500708577 and parameters: {'C': 9.042256345151335, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 25, C: 9.042256345151335, penalty: l1, ROC-AUC: 0.714705500708577\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:24,047] Trial 26 finished with value: 0.5736977904394385 and parameters: {'C': 6.266756642224746, 'penalty': 'l2'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 26, C: 6.266756642224746, penalty: l2, ROC-AUC: 0.5736977904394385\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:32,253] Trial 27 finished with value: 0.7140795371465828 and parameters: {'C': 3.421396553139845, 'penalty': 'l1'}. Best is trial 8 with value: 0.7147456063346245.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 27, C: 3.421396553139845, penalty: l1, ROC-AUC: 0.7140795371465828\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:34,932] Trial 28 finished with value: 0.7147574460330345 and parameters: {'C': 9.959148357262013, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 28, C: 9.959148357262013, penalty: l1, ROC-AUC: 0.7147574460330345\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:43,739] Trial 29 finished with value: 0.7147482882756683 and parameters: {'C': 9.958223914840454, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 29, C: 9.958223914840454, penalty: l1, ROC-AUC: 0.7147482882756683\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:44,892] Trial 30 finished with value: 0.682780633941202 and parameters: {'C': 0.11216062467361604, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 30, C: 0.11216062467361604, penalty: l1, ROC-AUC: 0.682780633941202\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:48:53,737] Trial 31 finished with value: 0.7145431836415487 and parameters: {'C': 6.147523642939506, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 31, C: 6.147523642939506, penalty: l1, ROC-AUC: 0.7145431836415487\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:01,804] Trial 32 finished with value: 0.7147192492186274 and parameters: {'C': 9.754409161643855, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 32, C: 9.754409161643855, penalty: l1, ROC-AUC: 0.7147192492186274\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:11,068] Trial 33 finished with value: 0.7145221846025722 and parameters: {'C': 5.806940812791451, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 33, C: 5.806940812791451, penalty: l1, ROC-AUC: 0.7145221846025722\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:17,044] Trial 34 finished with value: 0.7131610129745151 and parameters: {'C': 1.8904768914694223, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 34, C: 1.8904768914694223, penalty: l1, ROC-AUC: 0.7131610129745151\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:28,527] Trial 35 finished with value: 0.7142712547680182 and parameters: {'C': 4.217561870533169, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 35, C: 4.217561870533169, penalty: l1, ROC-AUC: 0.7142712547680182\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:37,502] Trial 36 finished with value: 0.7137953849124741 and parameters: {'C': 2.732176482666836, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 36, C: 2.732176482666836, penalty: l1, ROC-AUC: 0.7137953849124741\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:41,703] Trial 37 finished with value: 0.5590609927011095 and parameters: {'C': 0.36358923690323297, 'penalty': 'l2'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 37, C: 0.36358923690323297, penalty: l2, ROC-AUC: 0.5590609927011095\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:47,183] Trial 38 finished with value: 0.7124930201982986 and parameters: {'C': 1.3826282611535645, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 38, C: 1.3826282611535645, penalty: l1, ROC-AUC: 0.7124930201982986\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:49:51,796] Trial 39 finished with value: 0.5514260369211562 and parameters: {'C': 7.103172878720764, 'penalty': 'l2'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 39, C: 7.103172878720764, penalty: l2, ROC-AUC: 0.5514260369211562\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:04,826] Trial 40 finished with value: 0.7146268357152896 and parameters: {'C': 7.339759027651316, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 40, C: 7.339759027651316, penalty: l1, ROC-AUC: 0.7146268357152896\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:08,350] Trial 41 finished with value: 0.7147139106677418 and parameters: {'C': 8.865838558342752, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 41, C: 8.865838558342752, penalty: l1, ROC-AUC: 0.7147139106677418\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:09,896] Trial 42 finished with value: 0.7147494186805711 and parameters: {'C': 9.90077532535762, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 42, C: 9.90077532535762, penalty: l1, ROC-AUC: 0.7147494186805711\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:19,474] Trial 43 finished with value: 0.714746375242161 and parameters: {'C': 9.947738510884966, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 43, C: 9.947738510884966, penalty: l1, ROC-AUC: 0.714746375242161\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:30,086] Trial 44 finished with value: 0.7144729143586247 and parameters: {'C': 5.446939820606816, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 44, C: 5.446939820606816, penalty: l1, ROC-AUC: 0.7144729143586247\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:40,640] Trial 45 finished with value: 0.7142471927644968 and parameters: {'C': 4.119142047610541, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 45, C: 4.119142047610541, penalty: l1, ROC-AUC: 0.7142471927644968\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:50,444] Trial 46 finished with value: 0.7147295674616991 and parameters: {'C': 9.783153165552074, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 46, C: 9.783153165552074, penalty: l1, ROC-AUC: 0.7147295674616991\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:50:56,349] Trial 47 finished with value: 0.5896399942194197 and parameters: {'C': 7.188721629427061, 'penalty': 'l2'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 47, C: 7.188721629427061, penalty: l2, ROC-AUC: 0.5896399942194197\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:05,717] Trial 48 finished with value: 0.7144389188296664 and parameters: {'C': 5.157424879606246, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 48, C: 5.157424879606246, penalty: l1, ROC-AUC: 0.7144389188296664\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:12,914] Trial 49 finished with value: 0.7130781430016453 and parameters: {'C': 1.8078486103123101, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 49, C: 1.8078486103123101, penalty: l1, ROC-AUC: 0.7130781430016453\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:21,492] Trial 50 finished with value: 0.714638280141397 and parameters: {'C': 7.730892329145662, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 50, C: 7.730892329145662, penalty: l1, ROC-AUC: 0.714638280141397\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:34,208] Trial 51 finished with value: 0.7147398851770375 and parameters: {'C': 9.766658487905355, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 51, C: 9.766658487905355, penalty: l1, ROC-AUC: 0.7147398851770375\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:44,152] Trial 52 finished with value: 0.714664268899494 and parameters: {'C': 8.036058848274681, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 52, C: 8.036058848274681, penalty: l1, ROC-AUC: 0.714664268899494\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:47,547] Trial 53 finished with value: 0.7147352865083525 and parameters: {'C': 9.973935338025791, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 53, C: 9.973935338025791, penalty: l1, ROC-AUC: 0.7147352865083525\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:58,138] Trial 54 finished with value: 0.7145344037412835 and parameters: {'C': 5.931632281070659, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 54, C: 5.931632281070659, penalty: l1, ROC-AUC: 0.7145344037412835\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:51:59,743] Trial 55 finished with value: 0.6987307468613689 and parameters: {'C': 0.22416621812532442, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 55, C: 0.22416621812532442, penalty: l1, ROC-AUC: 0.6987307468613689\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:11,012] Trial 56 finished with value: 0.7146543343185904 and parameters: {'C': 8.070941636169355, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 56, C: 8.070941636169355, penalty: l1, ROC-AUC: 0.7146543343185904\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:13,909] Trial 57 finished with value: 0.7097939051269635 and parameters: {'C': 0.6897985310521529, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 57, C: 0.6897985310521529, penalty: l1, ROC-AUC: 0.7097939051269635\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:18,747] Trial 58 finished with value: 0.564169053393657 and parameters: {'C': 4.697102636998124, 'penalty': 'l2'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 58, C: 4.697102636998124, penalty: l2, ROC-AUC: 0.564169053393657\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:28,402] Trial 59 finished with value: 0.7145978077406496 and parameters: {'C': 6.660109777576608, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 59, C: 6.660109777576608, penalty: l1, ROC-AUC: 0.7145978077406496\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:37,366] Trial 60 finished with value: 0.7137846935619017 and parameters: {'C': 2.7087111421899213, 'penalty': 'l1'}. Best is trial 28 with value: 0.7147574460330345.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 60, C: 2.7087111421899213, penalty: l1, ROC-AUC: 0.7137846935619017\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:40,638] Trial 61 finished with value: 0.7147578275842691 and parameters: {'C': 9.835292207181045, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 61, C: 9.835292207181045, penalty: l1, ROC-AUC: 0.7147578275842691\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:47,310] Trial 62 finished with value: 0.7147379700325966 and parameters: {'C': 9.999504760944333, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 62, C: 9.999504760944333, penalty: l1, ROC-AUC: 0.7147379700325966\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:50,102] Trial 63 finished with value: 0.7146562346864966 and parameters: {'C': 8.061874199616243, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 63, C: 8.061874199616243, penalty: l1, ROC-AUC: 0.7146562346864966\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:52:58,229] Trial 64 finished with value: 0.7146604412492791 and parameters: {'C': 8.26016865319015, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 64, C: 8.26016865319015, penalty: l1, ROC-AUC: 0.7146604412492791\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:06,853] Trial 65 finished with value: 0.7145859622371724 and parameters: {'C': 6.495151977044344, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 65, C: 6.495151977044344, penalty: l1, ROC-AUC: 0.7145859622371724\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:17,565] Trial 66 finished with value: 0.7144885743189823 and parameters: {'C': 5.601842220782326, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 66, C: 5.601842220782326, penalty: l1, ROC-AUC: 0.7144885743189823\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:21,697] Trial 67 finished with value: 0.711808602776055 and parameters: {'C': 1.1000177256307049, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 67, C: 1.1000177256307049, penalty: l1, ROC-AUC: 0.711808602776055\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:26,925] Trial 68 finished with value: 0.7147020641087978 and parameters: {'C': 8.81152871739927, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 68, C: 8.81152871739927, penalty: l1, ROC-AUC: 0.7147020641087978\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:35,046] Trial 69 finished with value: 0.71417425420613 and parameters: {'C': 3.8081064617048432, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 69, C: 3.8081064617048432, penalty: l1, ROC-AUC: 0.71417425420613\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:39,184] Trial 70 finished with value: 0.5667419209191106 and parameters: {'C': 7.171284053348083, 'penalty': 'l2'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 70, C: 7.171284053348083, penalty: l2, ROC-AUC: 0.5667419209191106\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:49,808] Trial 71 finished with value: 0.7147280349239596 and parameters: {'C': 9.974227728264148, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 71, C: 9.974227728264148, penalty: l1, ROC-AUC: 0.7147280349239596\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:53:54,921] Trial 72 finished with value: 0.7147108503418635 and parameters: {'C': 8.89294127728949, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 72, C: 8.89294127728949, penalty: l1, ROC-AUC: 0.7147108503418635\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:54:06,212] Trial 73 finished with value: 0.7145871079463433 and parameters: {'C': 6.7061195794670665, 'penalty': 'l1'}. Best is trial 61 with value: 0.7147578275842691.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 73, C: 6.7061195794670665, penalty: l1, ROC-AUC: 0.7145871079463433\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:54:09,178] Trial 74 finished with value: 0.714758206496837 and parameters: {'C': 9.906911876359292, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 74, C: 9.906911876359292, penalty: l1, ROC-AUC: 0.714758206496837\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:54:23,247] Trial 75 finished with value: 0.7143529874754513 and parameters: {'C': 4.629570859218188, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 75, C: 4.629570859218188, penalty: l1, ROC-AUC: 0.7143529874754513\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:54:27,235] Trial 76 finished with value: 0.7147077952933192 and parameters: {'C': 8.76203703144043, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 76, C: 8.76203703144043, penalty: l1, ROC-AUC: 0.7147077952933192\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:54:36,803] Trial 77 finished with value: 0.7146314201351732 and parameters: {'C': 7.605845488400354, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 77, C: 7.605845488400354, penalty: l1, ROC-AUC: 0.7146314201351732\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:54:49,298] Trial 78 finished with value: 0.7145466265741284 and parameters: {'C': 6.073764959310895, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 78, C: 6.073764959310895, penalty: l1, ROC-AUC: 0.7145466265741284\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:00,080] Trial 79 finished with value: 0.7144847572234352 and parameters: {'C': 5.446456807423127, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 79, C: 5.446456807423127, penalty: l1, ROC-AUC: 0.7144847572234352\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:07,403] Trial 80 finished with value: 0.7146925084404622 and parameters: {'C': 8.7008946588877, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 80, C: 8.7008946588877, penalty: l1, ROC-AUC: 0.7146925084404622\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:13,397] Trial 81 finished with value: 0.7147303147321671 and parameters: {'C': 9.668326265544424, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 81, C: 9.668326265544424, penalty: l1, ROC-AUC: 0.7147303147321671\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:17,753] Trial 82 finished with value: 0.714747126734496 and parameters: {'C': 9.953257701347761, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 82, C: 9.953257701347761, penalty: l1, ROC-AUC: 0.714747126734496\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:24,710] Trial 83 finished with value: 0.7146214781660024 and parameters: {'C': 7.406777323204251, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 83, C: 7.406777323204251, penalty: l1, ROC-AUC: 0.7146214781660024\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:31,774] Trial 84 finished with value: 0.7146600649753783 and parameters: {'C': 8.155548020184362, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 84, C: 8.155548020184362, penalty: l1, ROC-AUC: 0.7146600649753783\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:40,816] Trial 85 finished with value: 0.7145706880499166 and parameters: {'C': 6.639507619029611, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 85, C: 6.639507619029611, penalty: l1, ROC-AUC: 0.7145706880499166\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:51,613] Trial 86 finished with value: 0.7147005479307933 and parameters: {'C': 8.744697864996457, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 86, C: 8.744697864996457, penalty: l1, ROC-AUC: 0.7147005479307933\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:55:55,979] Trial 87 finished with value: 0.5768890986875356 and parameters: {'C': 0.4472194377938711, 'penalty': 'l2'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 87, C: 0.4472194377938711, penalty: l2, ROC-AUC: 0.5768890986875356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:06,835] Trial 88 finished with value: 0.7146463080215938 and parameters: {'C': 7.468825413817064, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 88, C: 7.468825413817064, penalty: l1, ROC-AUC: 0.7146463080215938\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:14,994] Trial 89 finished with value: 0.7147215406369691 and parameters: {'C': 9.162223740619512, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 89, C: 9.162223740619512, penalty: l1, ROC-AUC: 0.7147215406369691\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:25,391] Trial 90 finished with value: 0.7144389262179336 and parameters: {'C': 5.0423179089922225, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 90, C: 5.0423179089922225, penalty: l1, ROC-AUC: 0.7144389262179336\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:28,726] Trial 91 finished with value: 0.7147234520872763 and parameters: {'C': 9.179692304328508, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 91, C: 9.179692304328508, penalty: l1, ROC-AUC: 0.7147234520872763\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:36,895] Trial 92 finished with value: 0.7147475082857306 and parameters: {'C': 9.714373342186786, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 92, C: 9.714373342186786, penalty: l1, ROC-AUC: 0.7147475082857306\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:45,525] Trial 93 finished with value: 0.7146470785123306 and parameters: {'C': 7.909902736640458, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 93, C: 7.909902736640458, penalty: l1, ROC-AUC: 0.7146470785123306\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:52,968] Trial 94 finished with value: 0.7145913166200593 and parameters: {'C': 6.790233781096166, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 94, C: 6.790233781096166, penalty: l1, ROC-AUC: 0.7145913166200593\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:56:54,391] Trial 95 finished with value: 0.7147471204016955 and parameters: {'C': 9.978600072076707, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 95, C: 9.978600072076707, penalty: l1, ROC-AUC: 0.7147471204016955\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:05,487] Trial 96 finished with value: 0.7145389839393 and parameters: {'C': 6.1150519659761535, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 96, C: 6.1150519659761535, penalty: l1, ROC-AUC: 0.7145389839393\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:13,619] Trial 97 finished with value: 0.7147356791419883 and parameters: {'C': 9.91813883818841, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 97, C: 9.91813883818841, penalty: l1, ROC-AUC: 0.7147356791419883\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:22,013] Trial 98 finished with value: 0.7146822081403255 and parameters: {'C': 8.220663005168845, 'penalty': 'l1'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 98, C: 8.220663005168845, penalty: l1, ROC-AUC: 0.7146822081403255\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:26,981] Trial 99 finished with value: 0.58144730183221 and parameters: {'C': 7.183908544205467, 'penalty': 'l2'}. Best is trial 74 with value: 0.714758206496837.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 99, C: 7.183908544205467, penalty: l2, ROC-AUC: 0.58144730183221\n","Best Parameters: {'C': 9.906911876359292, 'penalty': 'l1'}\n","Best ROC-AUC Score: 0.714758206496837\n"]}]},{"cell_type":"code","source":["# Decision Tree\n","print('\\nDecision Tree started.')\n","def objective_function(trial):\n"," max_depth = trial.suggest_int('max_depth', 1, 10)\n"," min_samples_split = trial.suggest_int('min_samples_split', 2, 10)\n"," min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10)\n"," criterion = trial.suggest_categorical('criterion', ['gini', 'entropy'])\n","\n"," model = DecisionTreeClassifier(\n"," max_depth=max_depth,\n"," min_samples_split=min_samples_split,\n"," min_samples_leaf=min_samples_leaf,\n"," criterion=criterion\n"," )\n"," metrics = ['roc_auc','accuracy','precison','recall','f1-score']\n"," # Using cross_val_score to get the average precision score for each fold\n"," scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc')\n"," roc_auc = np.mean(scores)\n"," # Printing intermediate results\n"," print(f\"Trial {trial.number}, max_depth: {max_depth}, min_samples_split: {min_samples_split}, \"\n"," f\"min_samples_leaf: {min_samples_leaf}, criterion: {criterion}, ROC-AUC: {roc_auc}\")\n"," return roc_auc\n","\n","study_dt = optuna.create_study(direction=\"maximize\")\n","study_dt.optimize(objective_function, n_trials=100)\n","\n","best_params_dt = study_dt.best_params\n","print(\"Best Parameters: \", best_params_dt)\n","print(\"Best ROC-AUC Score: \", study_dt.best_value)\n","\n","# Create and save model\n","best_model_dt = DecisionTreeClassifier(**best_params_dt)\n","with open('storage_files/best_models_dt_36_features.pkl', 'wb') as file:\n"," pickle.dump(best_model_dt, file)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rtkZ37BwDkEr","executionInfo":{"status":"ok","timestamp":1715572771689,"user_tz":-480,"elapsed":124137,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"6f3aef26-0414-4494-abbb-bc9f23b48d03"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:27,011] A new study created in memory with name: no-name-33744037-8943-44df-9a1c-dde5afab3132\n"]},{"output_type":"stream","name":"stdout","text":["\n","Decision Tree started.\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:27,479] Trial 0 finished with value: 0.733584655009923 and parameters: {'max_depth': 2, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 0 with value: 0.733584655009923.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 0, max_depth: 2, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.733584655009923\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:28,386] Trial 1 finished with value: 0.8779034183423151 and parameters: {'max_depth': 7, 'min_samples_split': 6, 'min_samples_leaf': 7, 'criterion': 'gini'}. Best is trial 1 with value: 0.8779034183423151.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 1, max_depth: 7, min_samples_split: 6, min_samples_leaf: 7, criterion: gini, ROC-AUC: 0.8779034183423151\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:29,522] Trial 2 finished with value: 0.9023758778475385 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 7, 'criterion': 'gini'}. Best is trial 2 with value: 0.9023758778475385.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 2, max_depth: 10, min_samples_split: 6, min_samples_leaf: 7, criterion: gini, ROC-AUC: 0.9023758778475385\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:30,567] Trial 3 finished with value: 0.9271549607381141 and parameters: {'max_depth': 7, 'min_samples_split': 3, 'min_samples_leaf': 4, 'criterion': 'entropy'}. Best is trial 3 with value: 0.9271549607381141.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 3, max_depth: 7, min_samples_split: 3, min_samples_leaf: 4, criterion: entropy, ROC-AUC: 0.9271549607381141\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:30,938] Trial 4 finished with value: 0.7657172558429901 and parameters: {'max_depth': 2, 'min_samples_split': 4, 'min_samples_leaf': 4, 'criterion': 'gini'}. Best is trial 3 with value: 0.9271549607381141.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 4, max_depth: 2, min_samples_split: 4, min_samples_leaf: 4, criterion: gini, ROC-AUC: 0.7657172558429901\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:32,210] Trial 5 finished with value: 0.9494770354813727 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 5 with value: 0.9494770354813727.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 5, max_depth: 10, min_samples_split: 6, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9494770354813727\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:32,788] Trial 6 finished with value: 0.7909104416186926 and parameters: {'max_depth': 3, 'min_samples_split': 3, 'min_samples_leaf': 4, 'criterion': 'entropy'}. Best is trial 5 with value: 0.9494770354813727.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 6, max_depth: 3, min_samples_split: 3, min_samples_leaf: 4, criterion: entropy, ROC-AUC: 0.7909104416186926\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:34,200] Trial 7 finished with value: 0.9484238464159918 and parameters: {'max_depth': 9, 'min_samples_split': 9, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 5 with value: 0.9494770354813727.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 7, max_depth: 9, min_samples_split: 9, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9484238464159918\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:35,516] Trial 8 finished with value: 0.8826562368924222 and parameters: {'max_depth': 8, 'min_samples_split': 5, 'min_samples_leaf': 10, 'criterion': 'gini'}. Best is trial 5 with value: 0.9494770354813727.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 8, max_depth: 8, min_samples_split: 5, min_samples_leaf: 10, criterion: gini, ROC-AUC: 0.8826562368924222\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:36,514] Trial 9 finished with value: 0.8250651352287237 and parameters: {'max_depth': 5, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'gini'}. Best is trial 5 with value: 0.9494770354813727.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 9, max_depth: 5, min_samples_split: 6, min_samples_leaf: 9, criterion: gini, ROC-AUC: 0.8250651352287237\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:37,556] Trial 10 finished with value: 0.8826236237610562 and parameters: {'max_depth': 5, 'min_samples_split': 10, 'min_samples_leaf': 1, 'criterion': 'entropy'}. Best is trial 5 with value: 0.9494770354813727.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 10, max_depth: 5, min_samples_split: 10, min_samples_leaf: 1, criterion: entropy, ROC-AUC: 0.8826236237610562\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:38,820] Trial 11 finished with value: 0.9501723746098097 and parameters: {'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 7, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 11, max_depth: 10, min_samples_split: 9, min_samples_leaf: 7, criterion: entropy, ROC-AUC: 0.9501723746098097\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:40,075] Trial 12 finished with value: 0.9493059005172757 and parameters: {'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 7, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 12, max_depth: 10, min_samples_split: 8, min_samples_leaf: 7, criterion: entropy, ROC-AUC: 0.9493059005172757\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:41,222] Trial 13 finished with value: 0.9386246127650768 and parameters: {'max_depth': 8, 'min_samples_split': 8, 'min_samples_leaf': 6, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 13, max_depth: 8, min_samples_split: 8, min_samples_leaf: 6, criterion: entropy, ROC-AUC: 0.9386246127650768\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:42,504] Trial 14 finished with value: 0.9498306208048625 and parameters: {'max_depth': 10, 'min_samples_split': 10, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 14, max_depth: 10, min_samples_split: 10, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9498306208048625\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:43,444] Trial 15 finished with value: 0.907860861289165 and parameters: {'max_depth': 6, 'min_samples_split': 10, 'min_samples_leaf': 5, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 15, max_depth: 6, min_samples_split: 10, min_samples_leaf: 5, criterion: entropy, ROC-AUC: 0.907860861289165\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:44,685] Trial 16 finished with value: 0.9447816651955572 and parameters: {'max_depth': 9, 'min_samples_split': 9, 'min_samples_leaf': 2, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 16, max_depth: 9, min_samples_split: 9, min_samples_leaf: 2, criterion: entropy, ROC-AUC: 0.9447816651955572\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:45,406] Trial 17 finished with value: 0.8407363861780632 and parameters: {'max_depth': 4, 'min_samples_split': 9, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 17, max_depth: 4, min_samples_split: 9, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.8407363861780632\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:46,558] Trial 18 finished with value: 0.9384005221309637 and parameters: {'max_depth': 8, 'min_samples_split': 10, 'min_samples_leaf': 6, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 18, max_depth: 8, min_samples_split: 10, min_samples_leaf: 6, criterion: entropy, ROC-AUC: 0.9384005221309637\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:48,050] Trial 19 finished with value: 0.9485588532805362 and parameters: {'max_depth': 9, 'min_samples_split': 8, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 19, max_depth: 9, min_samples_split: 8, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9485588532805362\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:49,577] Trial 20 finished with value: 0.9292948578545916 and parameters: {'max_depth': 7, 'min_samples_split': 9, 'min_samples_leaf': 5, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 20, max_depth: 7, min_samples_split: 9, min_samples_leaf: 5, criterion: entropy, ROC-AUC: 0.9292948578545916\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:51,230] Trial 21 finished with value: 0.9496823707349801 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 21, max_depth: 10, min_samples_split: 7, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9496823707349801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:52,481] Trial 22 finished with value: 0.9500588855461553 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 22, max_depth: 10, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9500588855461553\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:53,688] Trial 23 finished with value: 0.9493313905673274 and parameters: {'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 23, max_depth: 9, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9493313905673274\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:54,958] Trial 24 finished with value: 0.9493174993053973 and parameters: {'max_depth': 10, 'min_samples_split': 10, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 24, max_depth: 10, min_samples_split: 10, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9493174993053973\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:55,259] Trial 25 finished with value: 0.6200912404575778 and parameters: {'max_depth': 1, 'min_samples_split': 8, 'min_samples_leaf': 7, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 25, max_depth: 1, min_samples_split: 8, min_samples_leaf: 7, criterion: entropy, ROC-AUC: 0.6200912404575778\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:56,276] Trial 26 finished with value: 0.8836029979667911 and parameters: {'max_depth': 8, 'min_samples_split': 9, 'min_samples_leaf': 9, 'criterion': 'gini'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 26, max_depth: 8, min_samples_split: 9, min_samples_leaf: 9, criterion: gini, ROC-AUC: 0.8836029979667911\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:57,480] Trial 27 finished with value: 0.947959834465659 and parameters: {'max_depth': 9, 'min_samples_split': 2, 'min_samples_leaf': 6, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 27, max_depth: 9, min_samples_split: 2, min_samples_leaf: 6, criterion: entropy, ROC-AUC: 0.947959834465659\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:58,440] Trial 28 finished with value: 0.9082374481359465 and parameters: {'max_depth': 6, 'min_samples_split': 5, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 11 with value: 0.9501723746098097.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 28, max_depth: 6, min_samples_split: 5, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9082374481359465\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:57:59,705] Trial 29 finished with value: 0.9508654049047035 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 29 with value: 0.9508654049047035.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 29, max_depth: 10, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9508654049047035\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:00,963] Trial 30 finished with value: 0.9482545479640617 and parameters: {'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 29 with value: 0.9508654049047035.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 30, max_depth: 9, min_samples_split: 7, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9482545479640617\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:02,829] Trial 31 finished with value: 0.9495601476838648 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 29 with value: 0.9508654049047035.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 31, max_depth: 10, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9495601476838648\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:04,515] Trial 32 finished with value: 0.9501862080849346 and parameters: {'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 29 with value: 0.9508654049047035.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 32, max_depth: 10, min_samples_split: 5, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9501862080849346\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:05,724] Trial 33 finished with value: 0.9490730650771676 and parameters: {'max_depth': 9, 'min_samples_split': 5, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 29 with value: 0.9508654049047035.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 33, max_depth: 9, min_samples_split: 5, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9490730650771676\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:06,736] Trial 34 finished with value: 0.8933534215572669 and parameters: {'max_depth': 8, 'min_samples_split': 4, 'min_samples_leaf': 7, 'criterion': 'gini'}. Best is trial 29 with value: 0.9508654049047035.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 34, max_depth: 8, min_samples_split: 4, min_samples_leaf: 7, criterion: gini, ROC-AUC: 0.8933534215572669\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:07,992] Trial 35 finished with value: 0.9513659106738214 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 35 with value: 0.9513659106738214.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 35, max_depth: 10, min_samples_split: 6, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9513659106738214\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:09,119] Trial 36 finished with value: 0.9048028669369252 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 10, 'criterion': 'gini'}. Best is trial 35 with value: 0.9513659106738214.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 36, max_depth: 10, min_samples_split: 6, min_samples_leaf: 10, criterion: gini, ROC-AUC: 0.9048028669369252\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:10,346] Trial 37 finished with value: 0.9473372278157035 and parameters: {'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 7, 'criterion': 'entropy'}. Best is trial 35 with value: 0.9513659106738214.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 37, max_depth: 9, min_samples_split: 4, min_samples_leaf: 7, criterion: entropy, ROC-AUC: 0.9473372278157035\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:11,396] Trial 38 finished with value: 0.9289941973287149 and parameters: {'max_depth': 7, 'min_samples_split': 5, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 35 with value: 0.9513659106738214.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 38, max_depth: 7, min_samples_split: 5, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9289941973287149\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:12,682] Trial 39 finished with value: 0.9476850323477344 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 3, 'criterion': 'entropy'}. Best is trial 35 with value: 0.9513659106738214.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 39, max_depth: 10, min_samples_split: 6, min_samples_leaf: 3, criterion: entropy, ROC-AUC: 0.9476850323477344\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:13,189] Trial 40 finished with value: 0.812753675631337 and parameters: {'max_depth': 3, 'min_samples_split': 4, 'min_samples_leaf': 10, 'criterion': 'gini'}. Best is trial 35 with value: 0.9513659106738214.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 40, max_depth: 3, min_samples_split: 4, min_samples_leaf: 10, criterion: gini, ROC-AUC: 0.812753675631337\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:14,637] Trial 41 finished with value: 0.951398790310545 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 41, max_depth: 10, min_samples_split: 6, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.951398790310545\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:16,490] Trial 42 finished with value: 0.9494325728885293 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 42, max_depth: 10, min_samples_split: 6, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9494325728885293\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:18,114] Trial 43 finished with value: 0.9493286001770735 and parameters: {'max_depth': 9, 'min_samples_split': 5, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 43, max_depth: 9, min_samples_split: 5, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9493286001770735\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:19,611] Trial 44 finished with value: 0.9494672631784841 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 44, max_depth: 10, min_samples_split: 6, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9494672631784841\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:21,267] Trial 45 finished with value: 0.9377621750413099 and parameters: {'max_depth': 8, 'min_samples_split': 5, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 45, max_depth: 8, min_samples_split: 5, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9377621750413099\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:22,918] Trial 46 finished with value: 0.9486748401062846 and parameters: {'max_depth': 9, 'min_samples_split': 3, 'min_samples_leaf': 7, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 46, max_depth: 9, min_samples_split: 3, min_samples_leaf: 7, criterion: entropy, ROC-AUC: 0.9486748401062846\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:24,169] Trial 47 finished with value: 0.950119832418898 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 47, max_depth: 10, min_samples_split: 6, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.950119832418898\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:25,166] Trial 48 finished with value: 0.8830087018379433 and parameters: {'max_depth': 8, 'min_samples_split': 7, 'min_samples_leaf': 10, 'criterion': 'gini'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 48, max_depth: 8, min_samples_split: 7, min_samples_leaf: 10, criterion: gini, ROC-AUC: 0.8830087018379433\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:26,237] Trial 49 finished with value: 0.9295557499760514 and parameters: {'max_depth': 7, 'min_samples_split': 8, 'min_samples_leaf': 6, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 49, max_depth: 7, min_samples_split: 8, min_samples_leaf: 6, criterion: entropy, ROC-AUC: 0.9295557499760514\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:27,444] Trial 50 finished with value: 0.94801522641683 and parameters: {'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 50, max_depth: 9, min_samples_split: 6, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.94801522641683\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:29,101] Trial 51 finished with value: 0.9513111559607085 and parameters: {'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 41 with value: 0.951398790310545.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 51, max_depth: 10, min_samples_split: 5, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9513111559607085\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:30,949] Trial 52 finished with value: 0.9515189248568039 and parameters: {'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 52, max_depth: 10, min_samples_split: 4, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9515189248568039\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:32,403] Trial 53 finished with value: 0.9512257871702781 and parameters: {'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 53, max_depth: 10, min_samples_split: 4, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9512257871702781\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:33,601] Trial 54 finished with value: 0.948274298121991 and parameters: {'max_depth': 9, 'min_samples_split': 3, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 54, max_depth: 9, min_samples_split: 3, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.948274298121991\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:34,853] Trial 55 finished with value: 0.9499657651439533 and parameters: {'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 55, max_depth: 10, min_samples_split: 4, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9499657651439533\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:36,124] Trial 56 finished with value: 0.9498107962365264 and parameters: {'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 56, max_depth: 10, min_samples_split: 4, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9498107962365264\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:37,398] Trial 57 finished with value: 0.9502496764677734 and parameters: {'max_depth': 10, 'min_samples_split': 3, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 57, max_depth: 10, min_samples_split: 3, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9502496764677734\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:38,607] Trial 58 finished with value: 0.9476901621800675 and parameters: {'max_depth': 9, 'min_samples_split': 5, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 58, max_depth: 9, min_samples_split: 5, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9476901621800675\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:39,325] Trial 59 finished with value: 0.8407363861780632 and parameters: {'max_depth': 4, 'min_samples_split': 4, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 59, max_depth: 4, min_samples_split: 4, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.8407363861780632\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:40,559] Trial 60 finished with value: 0.9438105088540361 and parameters: {'max_depth': 9, 'min_samples_split': 3, 'min_samples_leaf': 1, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 60, max_depth: 9, min_samples_split: 3, min_samples_leaf: 1, criterion: entropy, ROC-AUC: 0.9438105088540361\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:41,982] Trial 61 finished with value: 0.9495315864894005 and parameters: {'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 61, max_depth: 10, min_samples_split: 2, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9495315864894005\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:43,816] Trial 62 finished with value: 0.9512300668241339 and parameters: {'max_depth': 10, 'min_samples_split': 3, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 62, max_depth: 10, min_samples_split: 3, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9512300668241339\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:45,495] Trial 63 finished with value: 0.949852908041023 and parameters: {'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 63, max_depth: 10, min_samples_split: 5, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.949852908041023\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:46,763] Trial 64 finished with value: 0.9498530850955724 and parameters: {'max_depth': 10, 'min_samples_split': 3, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 64, max_depth: 10, min_samples_split: 3, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9498530850955724\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:47,965] Trial 65 finished with value: 0.9482440297100382 and parameters: {'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 65, max_depth: 9, min_samples_split: 4, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9482440297100382\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:49,222] Trial 66 finished with value: 0.9491431016296703 and parameters: {'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 66, max_depth: 10, min_samples_split: 2, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9491431016296703\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:50,382] Trial 67 finished with value: 0.8906593563666579 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 8, 'criterion': 'gini'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 67, max_depth: 10, min_samples_split: 7, min_samples_leaf: 8, criterion: gini, ROC-AUC: 0.8906593563666579\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:51,592] Trial 68 finished with value: 0.9485777033891841 and parameters: {'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 68, max_depth: 9, min_samples_split: 4, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9485777033891841\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:52,714] Trial 69 finished with value: 0.9374278036986343 and parameters: {'max_depth': 8, 'min_samples_split': 6, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 69, max_depth: 8, min_samples_split: 6, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9374278036986343\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:53,029] Trial 70 finished with value: 0.6200912404575778 and parameters: {'max_depth': 1, 'min_samples_split': 5, 'min_samples_leaf': 3, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 70, max_depth: 1, min_samples_split: 5, min_samples_leaf: 3, criterion: entropy, ROC-AUC: 0.6200912404575778\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:54,298] Trial 71 finished with value: 0.9498509134727083 and parameters: {'max_depth': 10, 'min_samples_split': 3, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 71, max_depth: 10, min_samples_split: 3, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9498509134727083\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:55,770] Trial 72 finished with value: 0.9497133988191354 and parameters: {'max_depth': 10, 'min_samples_split': 3, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 72, max_depth: 10, min_samples_split: 3, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9497133988191354\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:57,633] Trial 73 finished with value: 0.9515033245303183 and parameters: {'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 73, max_depth: 10, min_samples_split: 2, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9515033245303183\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:58:59,285] Trial 74 finished with value: 0.9500370830331057 and parameters: {'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 74, max_depth: 10, min_samples_split: 2, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9500370830331057\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:00,485] Trial 75 finished with value: 0.9480518007149563 and parameters: {'max_depth': 9, 'min_samples_split': 2, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 75, max_depth: 9, min_samples_split: 2, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9480518007149563\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:01,697] Trial 76 finished with value: 0.9487668942231899 and parameters: {'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 76, max_depth: 9, min_samples_split: 6, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9487668942231899\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:02,964] Trial 77 finished with value: 0.9504079832888369 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 77, max_depth: 10, min_samples_split: 7, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9504079832888369\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:04,123] Trial 78 finished with value: 0.9003664738919953 and parameters: {'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 5, 'criterion': 'gini'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 78, max_depth: 10, min_samples_split: 2, min_samples_leaf: 5, criterion: gini, ROC-AUC: 0.9003664738919953\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:05,350] Trial 79 finished with value: 0.9485618835256119 and parameters: {'max_depth': 9, 'min_samples_split': 3, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 79, max_depth: 9, min_samples_split: 3, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9485618835256119\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:06,200] Trial 80 finished with value: 0.882706973179703 and parameters: {'max_depth': 5, 'min_samples_split': 5, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 80, max_depth: 5, min_samples_split: 5, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.882706973179703\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:07,486] Trial 81 finished with value: 0.9496679620304699 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 81, max_depth: 10, min_samples_split: 7, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9496679620304699\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:08,764] Trial 82 finished with value: 0.9485368937668059 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 82, max_depth: 10, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9485368937668059\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:10,592] Trial 83 finished with value: 0.9493055862520471 and parameters: {'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 83, max_depth: 10, min_samples_split: 6, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9493055862520471\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:12,388] Trial 84 finished with value: 0.9502848021377677 and parameters: {'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 84, max_depth: 10, min_samples_split: 8, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9502848021377677\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:13,671] Trial 85 finished with value: 0.9496394974112145 and parameters: {'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 85, max_depth: 10, min_samples_split: 7, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9496394974112145\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:14,137] Trial 86 finished with value: 0.733584655009923 and parameters: {'max_depth': 2, 'min_samples_split': 6, 'min_samples_leaf': 7, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 86, max_depth: 2, min_samples_split: 6, min_samples_leaf: 7, criterion: entropy, ROC-AUC: 0.733584655009923\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:15,341] Trial 87 finished with value: 0.9480120315189395 and parameters: {'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 87, max_depth: 9, min_samples_split: 4, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9480120315189395\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:16,605] Trial 88 finished with value: 0.9501854156932623 and parameters: {'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 88, max_depth: 10, min_samples_split: 4, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9501854156932623\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:17,821] Trial 89 finished with value: 0.9489526875096865 and parameters: {'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 89, max_depth: 9, min_samples_split: 6, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9489526875096865\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:18,653] Trial 90 finished with value: 0.8730881900425324 and parameters: {'max_depth': 6, 'min_samples_split': 5, 'min_samples_leaf': 9, 'criterion': 'gini'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 90, max_depth: 6, min_samples_split: 5, min_samples_leaf: 9, criterion: gini, ROC-AUC: 0.8730881900425324\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:19,927] Trial 91 finished with value: 0.9504371680002912 and parameters: {'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 91, max_depth: 10, min_samples_split: 8, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9504371680002912\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:21,185] Trial 92 finished with value: 0.9506491880526398 and parameters: {'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 92, max_depth: 10, min_samples_split: 8, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9506491880526398\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:22,553] Trial 93 finished with value: 0.9505190616664055 and parameters: {'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 93, max_depth: 10, min_samples_split: 8, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9505190616664055\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:24,394] Trial 94 finished with value: 0.9487446539553005 and parameters: {'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 94, max_depth: 10, min_samples_split: 8, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9487446539553005\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:26,166] Trial 95 finished with value: 0.9506943018684758 and parameters: {'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 95, max_depth: 10, min_samples_split: 9, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9506943018684758\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:27,379] Trial 96 finished with value: 0.9482709148232834 and parameters: {'max_depth': 9, 'min_samples_split': 9, 'min_samples_leaf': 10, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 96, max_depth: 9, min_samples_split: 9, min_samples_leaf: 10, criterion: entropy, ROC-AUC: 0.9482709148232834\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:28,635] Trial 97 finished with value: 0.9501313280351434 and parameters: {'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 97, max_depth: 10, min_samples_split: 9, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9501313280351434\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:29,847] Trial 98 finished with value: 0.9494186900703333 and parameters: {'max_depth': 9, 'min_samples_split': 10, 'min_samples_leaf': 8, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 98, max_depth: 9, min_samples_split: 10, min_samples_leaf: 8, criterion: entropy, ROC-AUC: 0.9494186900703333\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:31,121] Trial 99 finished with value: 0.9503641389356495 and parameters: {'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 9, 'criterion': 'entropy'}. Best is trial 52 with value: 0.9515189248568039.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 99, max_depth: 10, min_samples_split: 9, min_samples_leaf: 9, criterion: entropy, ROC-AUC: 0.9503641389356495\n","Best Parameters: {'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 9, 'criterion': 'entropy'}\n","Best ROC-AUC Score: 0.9515189248568039\n"]}]},{"cell_type":"code","source":["# KNN\n","print('\\nKNN started')\n","def objective_function(trial):\n"," n_neighbors = trial.suggest_int('n_neighbors', 1, 10)\n"," weights = trial.suggest_categorical('weights', ['uniform', 'distance'])\n"," p = trial.suggest_int('p', 1, 5)\n"," metric = trial.suggest_categorical('metric', ['euclidean', 'manhattan', 'minkowski'])\n","\n"," model = KNeighborsClassifier(\n"," n_neighbors=n_neighbors,\n"," weights=weights,\n"," p=p,\n"," metric=metric\n"," )\n"," metrics = ['roc_auc','accuracy','precison','recall','f1-score']\n"," # Using cross_val_score to get the average precision score for each fold\n"," scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc')\n"," roc_auc = np.mean(scores)\n"," # Printing intermediate results\n"," print(f\"Trial {trial.number}, n_neighbors: {n_neighbors}, weights: {weights}, p: {p}, metric: {metric}, \"\n"," f\"ROC-AUC: {roc_auc}\")\n"," return roc_auc\n","\n","\n","study_knn = optuna.create_study(direction=\"maximize\")\n","study_knn.optimize(objective_function, n_trials=100)\n","\n","best_params_knn = study_knn.best_params\n","print(\"Best Parameters: \", best_params_knn)\n","print(\"Best ROC-AUC Score: \", study_knn.best_value)\n","\n","# Create and save model\n","best_model_knn = KNeighborsClassifier(**best_params_knn)\n","with open('storage_files/best_models_knn_36_features.pkl', 'wb') as file:\n"," pickle.dump(best_model_knn, file)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GvJxSO19DoG3","executionInfo":{"status":"ok","timestamp":1715573292573,"user_tz":-480,"elapsed":520912,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"efe133cb-4fef-42f4-fec6-c0ba093dc2f5"},"execution_count":8,"outputs":[{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:31,157] A new study created in memory with name: no-name-95ae07e3-52fa-49c7-8bf7-4d50ef4963eb\n"]},{"output_type":"stream","name":"stdout","text":["\n","KNN started\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:31,573] Trial 0 finished with value: 0.8731433487355489 and parameters: {'n_neighbors': 3, 'weights': 'uniform', 'p': 5, 'metric': 'euclidean'}. Best is trial 0 with value: 0.8731433487355489.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 0, n_neighbors: 3, weights: uniform, p: 5, metric: euclidean, ROC-AUC: 0.8731433487355489\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:31,957] Trial 1 finished with value: 0.904412257950335 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'euclidean'}. Best is trial 1 with value: 0.904412257950335.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 1, n_neighbors: 9, weights: distance, p: 3, metric: euclidean, ROC-AUC: 0.904412257950335\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 03:59:32,375] Trial 2 finished with value: 0.9039314862432786 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 3, 'metric': 'euclidean'}. Best is trial 1 with value: 0.904412257950335.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 2, n_neighbors: 10, weights: distance, p: 3, metric: euclidean, ROC-AUC: 0.9039314862432786\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:08,178] Trial 3 finished with value: 0.8481811642625878 and parameters: {'n_neighbors': 4, 'weights': 'uniform', 'p': 5, 'metric': 'minkowski'}. Best is trial 1 with value: 0.904412257950335.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 3, n_neighbors: 4, weights: uniform, p: 5, metric: minkowski, ROC-AUC: 0.8481811642625878\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:08,551] Trial 4 finished with value: 0.8855865825666529 and parameters: {'n_neighbors': 7, 'weights': 'uniform', 'p': 2, 'metric': 'minkowski'}. Best is trial 1 with value: 0.904412257950335.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 4, n_neighbors: 7, weights: uniform, p: 2, metric: minkowski, ROC-AUC: 0.8855865825666529\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:08,928] Trial 5 finished with value: 0.8783404464932604 and parameters: {'n_neighbors': 3, 'weights': 'distance', 'p': 2, 'metric': 'euclidean'}. Best is trial 1 with value: 0.904412257950335.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 5, n_neighbors: 3, weights: distance, p: 2, metric: euclidean, ROC-AUC: 0.8783404464932604\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:43,365] Trial 6 finished with value: 0.8677063040665487 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'minkowski'}. Best is trial 1 with value: 0.904412257950335.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 6, n_neighbors: 9, weights: distance, p: 4, metric: minkowski, ROC-AUC: 0.8677063040665487\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:45,508] Trial 7 finished with value: 0.93663727070354 and parameters: {'n_neighbors': 8, 'weights': 'uniform', 'p': 2, 'metric': 'manhattan'}. Best is trial 7 with value: 0.93663727070354.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 7, n_neighbors: 8, weights: uniform, p: 2, metric: manhattan, ROC-AUC: 0.93663727070354\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:45,868] Trial 8 finished with value: 0.8731433487355489 and parameters: {'n_neighbors': 3, 'weights': 'uniform', 'p': 2, 'metric': 'euclidean'}. Best is trial 7 with value: 0.93663727070354.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 8, n_neighbors: 3, weights: uniform, p: 2, metric: euclidean, ROC-AUC: 0.8731433487355489\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:49,910] Trial 9 finished with value: 0.9407079782901464 and parameters: {'n_neighbors': 6, 'weights': 'distance', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 9, n_neighbors: 6, weights: distance, p: 1, metric: manhattan, ROC-AUC: 0.9407079782901464\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:52,035] Trial 10 finished with value: 0.8561363792668668 and parameters: {'n_neighbors': 1, 'weights': 'distance', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 10, n_neighbors: 1, weights: distance, p: 1, metric: manhattan, ROC-AUC: 0.8561363792668668\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:54,181] Trial 11 finished with value: 0.9355620960016259 and parameters: {'n_neighbors': 7, 'weights': 'uniform', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 11, n_neighbors: 7, weights: uniform, p: 1, metric: manhattan, ROC-AUC: 0.9355620960016259\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:56,327] Trial 12 finished with value: 0.9407079782901464 and parameters: {'n_neighbors': 6, 'weights': 'distance', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 12, n_neighbors: 6, weights: distance, p: 1, metric: manhattan, ROC-AUC: 0.9407079782901464\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:00:58,481] Trial 13 finished with value: 0.9344217443800511 and parameters: {'n_neighbors': 5, 'weights': 'distance', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 13, n_neighbors: 5, weights: distance, p: 1, metric: manhattan, ROC-AUC: 0.9344217443800511\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:01,308] Trial 14 finished with value: 0.9407079782901464 and parameters: {'n_neighbors': 6, 'weights': 'distance', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 14, n_neighbors: 6, weights: distance, p: 1, metric: manhattan, ROC-AUC: 0.9407079782901464\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:04,684] Trial 15 finished with value: 0.9344217443800511 and parameters: {'n_neighbors': 5, 'weights': 'distance', 'p': 1, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 15, n_neighbors: 5, weights: distance, p: 1, metric: manhattan, ROC-AUC: 0.9344217443800511\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:06,820] Trial 16 finished with value: 0.9407079782901464 and parameters: {'n_neighbors': 6, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 9 with value: 0.9407079782901464.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 16, n_neighbors: 6, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9407079782901464\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:08,976] Trial 17 finished with value: 0.9445246928137869 and parameters: {'n_neighbors': 7, 'weights': 'distance', 'p': 2, 'metric': 'manhattan'}. Best is trial 17 with value: 0.9445246928137869.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 17, n_neighbors: 7, weights: distance, p: 2, metric: manhattan, ROC-AUC: 0.9445246928137869\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:11,132] Trial 18 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 2, 'metric': 'manhattan'}. Best is trial 18 with value: 0.9477873086602356.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 18, n_neighbors: 8, weights: distance, p: 2, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:13,288] Trial 19 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 18 with value: 0.9477873086602356.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 19, n_neighbors: 8, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:49,413] Trial 20 finished with value: 0.866919119539759 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'minkowski'}. Best is trial 18 with value: 0.9477873086602356.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 20, n_neighbors: 10, weights: distance, p: 4, metric: minkowski, ROC-AUC: 0.866919119539759\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:51,561] Trial 21 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 18 with value: 0.9477873086602356.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 21, n_neighbors: 8, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:53,721] Trial 22 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 18 with value: 0.9477873086602356.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 22, n_neighbors: 8, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:55,879] Trial 23 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 18 with value: 0.9477873086602356.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 23, n_neighbors: 8, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:01:59,612] Trial 24 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 24, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:03,563] Trial 25 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 25, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:06,128] Trial 26 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 26, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:08,275] Trial 27 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 27, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:44,180] Trial 28 finished with value: 0.8619198911915091 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'minkowski'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 28, n_neighbors: 10, weights: distance, p: 5, metric: minkowski, ROC-AUC: 0.8619198911915091\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:46,333] Trial 29 finished with value: 0.9354241559949479 and parameters: {'n_neighbors': 9, 'weights': 'uniform', 'p': 4, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 29, n_neighbors: 9, weights: uniform, p: 4, metric: manhattan, ROC-AUC: 0.9354241559949479\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:46,730] Trial 30 finished with value: 0.9039314862432786 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'euclidean'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 30, n_neighbors: 10, weights: distance, p: 5, metric: euclidean, ROC-AUC: 0.9039314862432786\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:48,892] Trial 31 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 31, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:51,048] Trial 32 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 24 with value: 0.9491534718544801.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 32, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:54,046] Trial 33 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 33, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:57,185] Trial 34 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 34, n_neighbors: 10, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:57,587] Trial 35 finished with value: 0.9039314862432786 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'euclidean'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 35, n_neighbors: 10, weights: distance, p: 5, metric: euclidean, ROC-AUC: 0.9039314862432786\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:02:59,754] Trial 36 finished with value: 0.9338055705361471 and parameters: {'n_neighbors': 10, 'weights': 'uniform', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 36, n_neighbors: 10, weights: uniform, p: 5, metric: manhattan, ROC-AUC: 0.9338055705361471\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:03:34,328] Trial 37 finished with value: 0.879069723627717 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 3, 'metric': 'minkowski'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 37, n_neighbors: 10, weights: distance, p: 3, metric: minkowski, ROC-AUC: 0.879069723627717\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:03:35,910] Trial 38 finished with value: 0.9035666010926278 and parameters: {'n_neighbors': 7, 'weights': 'distance', 'p': 5, 'metric': 'euclidean'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 38, n_neighbors: 7, weights: distance, p: 5, metric: euclidean, ROC-AUC: 0.9035666010926278\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:03:38,853] Trial 39 finished with value: 0.8561363792668668 and parameters: {'n_neighbors': 1, 'weights': 'uniform', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 39, n_neighbors: 1, weights: uniform, p: 5, metric: manhattan, ROC-AUC: 0.8561363792668668\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:13,402] Trial 40 finished with value: 0.8677063040665487 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'minkowski'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 40, n_neighbors: 9, weights: distance, p: 4, metric: minkowski, ROC-AUC: 0.8677063040665487\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:15,541] Trial 41 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 41, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:19,585] Trial 42 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 42, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:21,779] Trial 43 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 43, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:23,931] Trial 44 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 44, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:26,081] Trial 45 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 45, n_neighbors: 10, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:26,479] Trial 46 finished with value: 0.874710561988344 and parameters: {'n_neighbors': 10, 'weights': 'uniform', 'p': 5, 'metric': 'euclidean'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 46, n_neighbors: 10, weights: uniform, p: 5, metric: euclidean, ROC-AUC: 0.874710561988344\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:28,608] Trial 47 finished with value: 0.8961210670300342 and parameters: {'n_neighbors': 2, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 47, n_neighbors: 2, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.8961210670300342\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:31,727] Trial 48 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 48, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:34,762] Trial 49 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 49, n_neighbors: 10, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:36,896] Trial 50 finished with value: 0.9262848650752508 and parameters: {'n_neighbors': 4, 'weights': 'distance', 'p': 2, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 50, n_neighbors: 4, weights: distance, p: 2, metric: manhattan, ROC-AUC: 0.9262848650752508\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:39,046] Trial 51 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 51, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:41,215] Trial 52 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 52, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:43,373] Trial 53 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 53, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:47,374] Trial 54 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 54, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:49,515] Trial 55 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 55, n_neighbors: 10, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:04:51,690] Trial 56 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 56, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:26,321] Trial 57 finished with value: 0.8299121594647316 and parameters: {'n_neighbors': 10, 'weights': 'uniform', 'p': 4, 'metric': 'minkowski'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 57, n_neighbors: 10, weights: uniform, p: 4, metric: minkowski, ROC-AUC: 0.8299121594647316\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:30,071] Trial 58 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 58, n_neighbors: 8, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:32,233] Trial 59 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 59, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:34,403] Trial 60 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 60, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:36,562] Trial 61 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 61, n_neighbors: 10, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:38,717] Trial 62 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 62, n_neighbors: 10, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:42,112] Trial 63 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 63, n_neighbors: 9, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:46,479] Trial 64 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 64, n_neighbors: 10, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:48,634] Trial 65 finished with value: 0.9477873086602356 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 65, n_neighbors: 8, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9477873086602356\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:50,798] Trial 66 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 66, n_neighbors: 10, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:51,198] Trial 67 finished with value: 0.8903926652469373 and parameters: {'n_neighbors': 4, 'weights': 'distance', 'p': 4, 'metric': 'euclidean'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 67, n_neighbors: 4, weights: distance, p: 4, metric: euclidean, ROC-AUC: 0.8903926652469373\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:53,346] Trial 68 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 68, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:56,379] Trial 69 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 69, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:05:59,567] Trial 70 finished with value: 0.9354241559949479 and parameters: {'n_neighbors': 9, 'weights': 'uniform', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 70, n_neighbors: 9, weights: uniform, p: 5, metric: manhattan, ROC-AUC: 0.9354241559949479\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:01,724] Trial 71 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 71, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:03,889] Trial 72 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 72, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:06,054] Trial 73 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 73, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:08,204] Trial 74 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 74, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:44,009] Trial 75 finished with value: 0.866919119539759 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'minkowski'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 75, n_neighbors: 10, weights: distance, p: 4, metric: minkowski, ROC-AUC: 0.866919119539759\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:46,167] Trial 76 finished with value: 0.9344217443800511 and parameters: {'n_neighbors': 5, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 76, n_neighbors: 5, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9344217443800511\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:48,364] Trial 77 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 77, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:50,519] Trial 78 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 2, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 78, n_neighbors: 9, weights: distance, p: 2, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:50,914] Trial 79 finished with value: 0.9039314862432786 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'euclidean'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 79, n_neighbors: 10, weights: distance, p: 4, metric: euclidean, ROC-AUC: 0.9039314862432786\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:54,985] Trial 80 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 80, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:57,161] Trial 81 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 81, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:06:59,322] Trial 82 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 82, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:01,483] Trial 83 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 83, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:03,660] Trial 84 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 84, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:06,845] Trial 85 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 85, n_neighbors: 9, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:09,624] Trial 86 finished with value: 0.9354241559949479 and parameters: {'n_neighbors': 9, 'weights': 'uniform', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 86, n_neighbors: 9, weights: uniform, p: 5, metric: manhattan, ROC-AUC: 0.9354241559949479\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:44,032] Trial 87 finished with value: 0.8809524591700072 and parameters: {'n_neighbors': 8, 'weights': 'distance', 'p': 3, 'metric': 'minkowski'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 87, n_neighbors: 8, weights: distance, p: 3, metric: minkowski, ROC-AUC: 0.8809524591700072\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:46,183] Trial 88 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 88, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:50,147] Trial 89 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 89, n_neighbors: 10, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:52,437] Trial 90 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 90, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:54,589] Trial 91 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 91, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:56,763] Trial 92 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 92, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:07:58,936] Trial 93 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 93, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:01,767] Trial 94 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 5, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 94, n_neighbors: 9, weights: distance, p: 5, metric: manhattan, ROC-AUC: 0.9491534718544801\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:05,150] Trial 95 finished with value: 0.8961210670300342 and parameters: {'n_neighbors': 2, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 95, n_neighbors: 2, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.8961210670300342\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:07,314] Trial 96 finished with value: 0.9495353969882805 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 96, n_neighbors: 10, weights: distance, p: 4, metric: manhattan, ROC-AUC: 0.9495353969882805\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:07,713] Trial 97 finished with value: 0.9039314862432786 and parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 5, 'metric': 'euclidean'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 97, n_neighbors: 10, weights: distance, p: 5, metric: euclidean, ROC-AUC: 0.9039314862432786\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:09,866] Trial 98 finished with value: 0.9355620960016259 and parameters: {'n_neighbors': 7, 'weights': 'uniform', 'p': 4, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 98, n_neighbors: 7, weights: uniform, p: 4, metric: manhattan, ROC-AUC: 0.9355620960016259\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:12,052] Trial 99 finished with value: 0.9491534718544801 and parameters: {'n_neighbors': 9, 'weights': 'distance', 'p': 3, 'metric': 'manhattan'}. Best is trial 33 with value: 0.9495353969882805.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 99, n_neighbors: 9, weights: distance, p: 3, metric: manhattan, ROC-AUC: 0.9491534718544801\n","Best Parameters: {'n_neighbors': 10, 'weights': 'distance', 'p': 4, 'metric': 'manhattan'}\n","Best ROC-AUC Score: 0.9495353969882805\n"]}]},{"cell_type":"code","source":["# Random Forest\n","print('\\nRandom Forest started')\n","def objective_function(trial):\n"," n_estimators = trial.suggest_int('n_estimators', 2, 50)\n"," max_depth = trial.suggest_int('max_depth', 1, 10)\n"," min_samples_split = trial.suggest_int('min_samples_split', 2, 10)\n"," min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10)\n","\n"," model = RandomForestClassifier(\n"," n_estimators=n_estimators,\n"," max_depth=max_depth,\n"," min_samples_split=min_samples_split,\n"," min_samples_leaf=min_samples_leaf,\n"," n_jobs=-1\n"," )\n","\n"," # Using cross_val_score to get the average ROC-AUC score for each fold\n"," scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc')\n"," roc_auc = np.mean(scores)\n"," # Printing intermediate results\n"," print(f\"Trial {trial.number}, n_estimators: {n_estimators}, max_depth: {max_depth}, \"\n"," f\"min_samples_split: {min_samples_split}, min_samples_leaf: {min_samples_leaf}, ROC-AUC: {roc_auc}\")\n"," return roc_auc\n","\n","\n","study_rf = optuna.create_study(direction=\"maximize\")\n","study_rf.optimize(objective_function, n_trials=100)\n","\n","best_params_rf = study_rf.best_params\n","print(\"Best Parameters: \", best_params_rf)\n","print(\"Best ROC-AUC: Score: \", study_rf.best_value)\n","\n","# Create and save model\n","best_model_rf = RandomForestClassifier(**best_params_rf, n_jobs=-1)\n","with open('storage_files/best_models_rf_36_features.pkl', 'wb') as file:\n"," pickle.dump(best_model_rf, file)\n","\n","label_encoder = LabelEncoder()\n","# Fit the encoder and transform the target variable\n","y_train_oversampled_encoded = label_encoder.fit_transform(y_train_oversampled)\n","# This suppresses printing logs\n","optuna.logging.set_verbosity(optuna.logging.WARNING)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fat9zn9yDsnS","executionInfo":{"status":"ok","timestamp":1715573686413,"user_tz":-480,"elapsed":393862,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"2706c440-f2c4-47a0-9834-7b27d549289f"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:12,078] A new study created in memory with name: no-name-fdfe4a8f-02e1-4a69-8e99-8fe79e399ac2\n"]},{"output_type":"stream","name":"stdout","text":["\n","Random Forest started\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:12,561] Trial 0 finished with value: 0.9294750917337827 and parameters: {'n_estimators': 3, 'max_depth': 8, 'min_samples_split': 9, 'min_samples_leaf': 10}. Best is trial 0 with value: 0.9294750917337827.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 0, n_estimators: 3, max_depth: 8, min_samples_split: 9, min_samples_leaf: 10, ROC-AUC: 0.9294750917337827\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:13,271] Trial 1 finished with value: 0.8686899035696012 and parameters: {'n_estimators': 15, 'max_depth': 2, 'min_samples_split': 6, 'min_samples_leaf': 3}. Best is trial 0 with value: 0.9294750917337827.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 1, n_estimators: 15, max_depth: 2, min_samples_split: 6, min_samples_leaf: 3, ROC-AUC: 0.8686899035696012\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:17,511] Trial 2 finished with value: 0.967552805952056 and parameters: {'n_estimators': 37, 'max_depth': 7, 'min_samples_split': 4, 'min_samples_leaf': 4}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 2, n_estimators: 37, max_depth: 7, min_samples_split: 4, min_samples_leaf: 4, ROC-AUC: 0.967552805952056\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:21,086] Trial 3 finished with value: 0.9528299306773652 and parameters: {'n_estimators': 49, 'max_depth': 6, 'min_samples_split': 3, 'min_samples_leaf': 3}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 3, n_estimators: 49, max_depth: 6, min_samples_split: 3, min_samples_leaf: 3, ROC-AUC: 0.9528299306773652\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:23,556] Trial 4 finished with value: 0.9346494766383622 and parameters: {'n_estimators': 47, 'max_depth': 4, 'min_samples_split': 3, 'min_samples_leaf': 10}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 4, n_estimators: 47, max_depth: 4, min_samples_split: 3, min_samples_leaf: 10, ROC-AUC: 0.9346494766383622\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:24,616] Trial 5 finished with value: 0.946976401990222 and parameters: {'n_estimators': 12, 'max_depth': 7, 'min_samples_split': 10, 'min_samples_leaf': 10}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 5, n_estimators: 12, max_depth: 7, min_samples_split: 10, min_samples_leaf: 10, ROC-AUC: 0.946976401990222\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:26,860] Trial 6 finished with value: 0.9563790083830659 and parameters: {'n_estimators': 32, 'max_depth': 6, 'min_samples_split': 2, 'min_samples_leaf': 8}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 6, n_estimators: 32, max_depth: 6, min_samples_split: 2, min_samples_leaf: 8, ROC-AUC: 0.9563790083830659\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:28,702] Trial 7 finished with value: 0.9355071713583643 and parameters: {'n_estimators': 29, 'max_depth': 4, 'min_samples_split': 9, 'min_samples_leaf': 3}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 7, n_estimators: 29, max_depth: 4, min_samples_split: 9, min_samples_leaf: 3, ROC-AUC: 0.9355071713583643\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:29,148] Trial 8 finished with value: 0.8055671475710987 and parameters: {'n_estimators': 2, 'max_depth': 3, 'min_samples_split': 2, 'min_samples_leaf': 2}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 8, n_estimators: 2, max_depth: 3, min_samples_split: 2, min_samples_leaf: 2, ROC-AUC: 0.8055671475710987\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:32,923] Trial 9 finished with value: 0.9474718277978113 and parameters: {'n_estimators': 44, 'max_depth': 5, 'min_samples_split': 8, 'min_samples_leaf': 4}. Best is trial 2 with value: 0.967552805952056.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 9, n_estimators: 44, max_depth: 5, min_samples_split: 8, min_samples_leaf: 4, ROC-AUC: 0.9474718277978113\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:36,405] Trial 10 finished with value: 0.978869373871363 and parameters: {'n_estimators': 37, 'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 6}. Best is trial 10 with value: 0.978869373871363.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 10, n_estimators: 37, max_depth: 10, min_samples_split: 5, min_samples_leaf: 6, ROC-AUC: 0.978869373871363\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:39,831] Trial 11 finished with value: 0.9801789426739271 and parameters: {'n_estimators': 37, 'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 6}. Best is trial 11 with value: 0.9801789426739271.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 11, n_estimators: 37, max_depth: 10, min_samples_split: 5, min_samples_leaf: 6, ROC-AUC: 0.9801789426739271\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:44,704] Trial 12 finished with value: 0.9815284424227892 and parameters: {'n_estimators': 38, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 12, n_estimators: 38, max_depth: 10, min_samples_split: 6, min_samples_leaf: 7, ROC-AUC: 0.9815284424227892\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:46,952] Trial 13 finished with value: 0.9775542248160247 and parameters: {'n_estimators': 20, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 13, n_estimators: 20, max_depth: 10, min_samples_split: 7, min_samples_leaf: 7, ROC-AUC: 0.9775542248160247\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:50,261] Trial 14 finished with value: 0.9739127304405859 and parameters: {'n_estimators': 38, 'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 8}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 14, n_estimators: 38, max_depth: 9, min_samples_split: 6, min_samples_leaf: 8, ROC-AUC: 0.9739127304405859\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:52,591] Trial 15 finished with value: 0.9741882230976404 and parameters: {'n_estimators': 25, 'max_depth': 9, 'min_samples_split': 5, 'min_samples_leaf': 6}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 15, n_estimators: 25, max_depth: 9, min_samples_split: 5, min_samples_leaf: 6, ROC-AUC: 0.9741882230976404\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:56,676] Trial 16 finished with value: 0.9750721928710988 and parameters: {'n_estimators': 41, 'max_depth': 8, 'min_samples_split': 7, 'min_samples_leaf': 5}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 16, n_estimators: 41, max_depth: 8, min_samples_split: 7, min_samples_leaf: 5, ROC-AUC: 0.9750721928710988\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:08:58,140] Trial 17 finished with value: 0.8657807933589018 and parameters: {'n_estimators': 32, 'max_depth': 1, 'min_samples_split': 5, 'min_samples_leaf': 8}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 17, n_estimators: 32, max_depth: 1, min_samples_split: 5, min_samples_leaf: 8, ROC-AUC: 0.8657807933589018\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:00,942] Trial 18 finished with value: 0.974781969064438 and parameters: {'n_estimators': 26, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 18, n_estimators: 26, max_depth: 10, min_samples_split: 6, min_samples_leaf: 7, ROC-AUC: 0.974781969064438\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:04,809] Trial 19 finished with value: 0.9792580130931073 and parameters: {'n_estimators': 43, 'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 1}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 19, n_estimators: 43, max_depth: 9, min_samples_split: 4, min_samples_leaf: 1, ROC-AUC: 0.9792580130931073\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:07,558] Trial 20 finished with value: 0.9723926846780804 and parameters: {'n_estimators': 33, 'max_depth': 8, 'min_samples_split': 7, 'min_samples_leaf': 5}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 20, n_estimators: 33, max_depth: 8, min_samples_split: 7, min_samples_leaf: 5, ROC-AUC: 0.9723926846780804\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:13,166] Trial 21 finished with value: 0.9804978614554642 and parameters: {'n_estimators': 44, 'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 1}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 21, n_estimators: 44, max_depth: 9, min_samples_split: 4, min_samples_leaf: 1, ROC-AUC: 0.9804978614554642\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:17,768] Trial 22 finished with value: 0.9811416887924164 and parameters: {'n_estimators': 50, 'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 22, n_estimators: 50, max_depth: 10, min_samples_split: 4, min_samples_leaf: 7, ROC-AUC: 0.9811416887924164\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:22,070] Trial 23 finished with value: 0.9762033371283696 and parameters: {'n_estimators': 50, 'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 9}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 23, n_estimators: 50, max_depth: 9, min_samples_split: 4, min_samples_leaf: 9, ROC-AUC: 0.9762033371283696\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:28,019] Trial 24 finished with value: 0.9634337920265228 and parameters: {'n_estimators': 45, 'max_depth': 7, 'min_samples_split': 3, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 24, n_estimators: 45, max_depth: 7, min_samples_split: 3, min_samples_leaf: 7, ROC-AUC: 0.9634337920265228\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:32,654] Trial 25 finished with value: 0.9803477714431477 and parameters: {'n_estimators': 40, 'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 25, n_estimators: 40, max_depth: 10, min_samples_split: 4, min_samples_leaf: 7, ROC-AUC: 0.9803477714431477\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:36,791] Trial 26 finished with value: 0.9745628404434431 and parameters: {'n_estimators': 50, 'max_depth': 8, 'min_samples_split': 3, 'min_samples_leaf': 1}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 26, n_estimators: 50, max_depth: 8, min_samples_split: 3, min_samples_leaf: 1, ROC-AUC: 0.9745628404434431\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:42,301] Trial 27 finished with value: 0.9746312922126016 and parameters: {'n_estimators': 46, 'max_depth': 9, 'min_samples_split': 5, 'min_samples_leaf': 9}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 27, n_estimators: 46, max_depth: 9, min_samples_split: 5, min_samples_leaf: 9, ROC-AUC: 0.9746312922126016\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:46,220] Trial 28 finished with value: 0.9797860868624658 and parameters: {'n_estimators': 42, 'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 5}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 28, n_estimators: 42, max_depth: 10, min_samples_split: 4, min_samples_leaf: 5, ROC-AUC: 0.9797860868624658\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:50,097] Trial 29 finished with value: 0.9740727803650083 and parameters: {'n_estimators': 47, 'max_depth': 8, 'min_samples_split': 8, 'min_samples_leaf': 9}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 29, n_estimators: 47, max_depth: 8, min_samples_split: 8, min_samples_leaf: 9, ROC-AUC: 0.9740727803650083\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:51,153] Trial 30 finished with value: 0.9666702307140576 and parameters: {'n_estimators': 9, 'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 4}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 30, n_estimators: 9, max_depth: 9, min_samples_split: 6, min_samples_leaf: 4, ROC-AUC: 0.9666702307140576\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:56,441] Trial 31 finished with value: 0.9801264020662155 and parameters: {'n_estimators': 40, 'max_depth': 10, 'min_samples_split': 4, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 31, n_estimators: 40, max_depth: 10, min_samples_split: 4, min_samples_leaf: 7, ROC-AUC: 0.9801264020662155\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:09:59,608] Trial 32 finished with value: 0.9799719244785667 and parameters: {'n_estimators': 34, 'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 8}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 32, n_estimators: 34, max_depth: 10, min_samples_split: 2, min_samples_leaf: 8, ROC-AUC: 0.9799719244785667\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:03,239] Trial 33 finished with value: 0.9759562032800277 and parameters: {'n_estimators': 41, 'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 6}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 33, n_estimators: 41, max_depth: 9, min_samples_split: 4, min_samples_leaf: 6, ROC-AUC: 0.9759562032800277\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:08,146] Trial 34 finished with value: 0.9738224880323789 and parameters: {'n_estimators': 47, 'max_depth': 8, 'min_samples_split': 3, 'min_samples_leaf': 7}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 34, n_estimators: 47, max_depth: 8, min_samples_split: 3, min_samples_leaf: 7, ROC-AUC: 0.9738224880323789\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:12,310] Trial 35 finished with value: 0.980961171784266 and parameters: {'n_estimators': 39, 'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 2}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 35, n_estimators: 39, max_depth: 10, min_samples_split: 5, min_samples_leaf: 2, ROC-AUC: 0.980961171784266\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:14,000] Trial 36 finished with value: 0.9619927648386701 and parameters: {'n_estimators': 20, 'max_depth': 7, 'min_samples_split': 6, 'min_samples_leaf': 2}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 36, n_estimators: 20, max_depth: 7, min_samples_split: 6, min_samples_leaf: 2, ROC-AUC: 0.9619927648386701\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:17,937] Trial 37 finished with value: 0.9808343527031328 and parameters: {'n_estimators': 44, 'max_depth': 9, 'min_samples_split': 5, 'min_samples_leaf': 2}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 37, n_estimators: 44, max_depth: 9, min_samples_split: 5, min_samples_leaf: 2, ROC-AUC: 0.9808343527031328\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:20,753] Trial 38 finished with value: 0.9576647465931961 and parameters: {'n_estimators': 29, 'max_depth': 6, 'min_samples_split': 5, 'min_samples_leaf': 2}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 38, n_estimators: 29, max_depth: 6, min_samples_split: 5, min_samples_leaf: 2, ROC-AUC: 0.9576647465931961\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:25,026] Trial 39 finished with value: 0.9811000458790293 and parameters: {'n_estimators': 35, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 2}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 39, n_estimators: 35, max_depth: 10, min_samples_split: 7, min_samples_leaf: 2, ROC-AUC: 0.9811000458790293\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:27,269] Trial 40 finished with value: 0.9387567236926916 and parameters: {'n_estimators': 35, 'max_depth': 5, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 12 with value: 0.9815284424227892.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 40, n_estimators: 35, max_depth: 5, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9387567236926916\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:31,811] Trial 41 finished with value: 0.9828458776191354 and parameters: {'n_estimators': 48, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 2}. Best is trial 41 with value: 0.9828458776191354.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 41, n_estimators: 48, max_depth: 10, min_samples_split: 7, min_samples_leaf: 2, ROC-AUC: 0.9828458776191354\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:37,888] Trial 42 finished with value: 0.9812686060319589 and parameters: {'n_estimators': 49, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 3}. Best is trial 41 with value: 0.9828458776191354.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 42, n_estimators: 49, max_depth: 10, min_samples_split: 7, min_samples_leaf: 3, ROC-AUC: 0.9812686060319589\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:42,517] Trial 43 finished with value: 0.9833749788853874 and parameters: {'n_estimators': 49, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 3}. Best is trial 43 with value: 0.9833749788853874.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 43, n_estimators: 49, max_depth: 10, min_samples_split: 7, min_samples_leaf: 3, ROC-AUC: 0.9833749788853874\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:44,712] Trial 44 finished with value: 0.918770001359019 and parameters: {'n_estimators': 48, 'max_depth': 3, 'min_samples_split': 9, 'min_samples_leaf': 3}. Best is trial 43 with value: 0.9833749788853874.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 44, n_estimators: 48, max_depth: 3, min_samples_split: 9, min_samples_leaf: 3, ROC-AUC: 0.918770001359019\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:50,921] Trial 45 finished with value: 0.9837504089405972 and parameters: {'n_estimators': 50, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 45, n_estimators: 50, max_depth: 10, min_samples_split: 8, min_samples_leaf: 4, ROC-AUC: 0.9837504089405972\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:54,882] Trial 46 finished with value: 0.9751574782796549 and parameters: {'n_estimators': 48, 'max_depth': 8, 'min_samples_split': 8, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 46, n_estimators: 48, max_depth: 8, min_samples_split: 8, min_samples_leaf: 4, ROC-AUC: 0.9751574782796549\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:10:58,994] Trial 47 finished with value: 0.9775817846363978 and parameters: {'n_estimators': 46, 'max_depth': 9, 'min_samples_split': 10, 'min_samples_leaf': 3}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 47, n_estimators: 46, max_depth: 9, min_samples_split: 10, min_samples_leaf: 3, ROC-AUC: 0.9775817846363978\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:05,315] Trial 48 finished with value: 0.9818544534132381 and parameters: {'n_estimators': 49, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 48, n_estimators: 49, max_depth: 10, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9818544534132381\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:09,319] Trial 49 finished with value: 0.981180196441785 and parameters: {'n_estimators': 43, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 49, n_estimators: 43, max_depth: 10, min_samples_split: 8, min_samples_leaf: 4, ROC-AUC: 0.981180196441785\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:12,741] Trial 50 finished with value: 0.9620327353649719 and parameters: {'n_estimators': 45, 'max_depth': 7, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 50, n_estimators: 45, max_depth: 7, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9620327353649719\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:19,045] Trial 51 finished with value: 0.9824241790674512 and parameters: {'n_estimators': 49, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 3}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 51, n_estimators: 49, max_depth: 10, min_samples_split: 7, min_samples_leaf: 3, ROC-AUC: 0.9824241790674512\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:23,237] Trial 52 finished with value: 0.98064754247441 and parameters: {'n_estimators': 48, 'max_depth': 9, 'min_samples_split': 9, 'min_samples_leaf': 5}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 52, n_estimators: 48, max_depth: 9, min_samples_split: 9, min_samples_leaf: 5, ROC-AUC: 0.98064754247441\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:27,884] Trial 53 finished with value: 0.9789458566871186 and parameters: {'n_estimators': 50, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 3}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 53, n_estimators: 50, max_depth: 10, min_samples_split: 6, min_samples_leaf: 3, ROC-AUC: 0.9789458566871186\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:33,862] Trial 54 finished with value: 0.9830003288201153 and parameters: {'n_estimators': 46, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 54, n_estimators: 46, max_depth: 10, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9830003288201153\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:37,922] Trial 55 finished with value: 0.9791283495290486 and parameters: {'n_estimators': 46, 'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 45 with value: 0.9837504089405972.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 55, n_estimators: 46, max_depth: 9, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9791283495290486\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:42,489] Trial 56 finished with value: 0.9844000197076754 and parameters: {'n_estimators': 48, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 56, n_estimators: 48, max_depth: 10, min_samples_split: 8, min_samples_leaf: 4, ROC-AUC: 0.9844000197076754\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:47,966] Trial 57 finished with value: 0.9822765240169561 and parameters: {'n_estimators': 42, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 57, n_estimators: 42, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9822765240169561\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:50,406] Trial 58 finished with value: 0.9282617566965039 and parameters: {'n_estimators': 45, 'max_depth': 4, 'min_samples_split': 8, 'min_samples_leaf': 5}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 58, n_estimators: 45, max_depth: 4, min_samples_split: 8, min_samples_leaf: 5, ROC-AUC: 0.9282617566965039\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:51,632] Trial 59 finished with value: 0.8573347319534499 and parameters: {'n_estimators': 47, 'max_depth': 1, 'min_samples_split': 9, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 59, n_estimators: 47, max_depth: 1, min_samples_split: 9, min_samples_leaf: 4, ROC-AUC: 0.8573347319534499\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:11:56,113] Trial 60 finished with value: 0.9793058853713278 and parameters: {'n_estimators': 50, 'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 2}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 60, n_estimators: 50, max_depth: 9, min_samples_split: 7, min_samples_leaf: 2, ROC-AUC: 0.9793058853713278\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:01,696] Trial 61 finished with value: 0.9829651173934464 and parameters: {'n_estimators': 43, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 61, n_estimators: 43, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9829651173934464\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:05,732] Trial 62 finished with value: 0.979887210077529 and parameters: {'n_estimators': 43, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 62, n_estimators: 43, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.979887210077529\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:10,596] Trial 63 finished with value: 0.9797046211881237 and parameters: {'n_estimators': 48, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 63, n_estimators: 48, max_depth: 10, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9797046211881237\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:15,883] Trial 64 finished with value: 0.9795660420963207 and parameters: {'n_estimators': 45, 'max_depth': 9, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 64, n_estimators: 45, max_depth: 9, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9795660420963207\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:20,402] Trial 65 finished with value: 0.9777425338075503 and parameters: {'n_estimators': 47, 'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 65, n_estimators: 47, max_depth: 10, min_samples_split: 9, min_samples_leaf: 1, ROC-AUC: 0.9777425338075503\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:21,357] Trial 66 finished with value: 0.973545342778241 and parameters: {'n_estimators': 8, 'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 5}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 66, n_estimators: 8, max_depth: 9, min_samples_split: 7, min_samples_leaf: 5, ROC-AUC: 0.973545342778241\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:27,792] Trial 67 finished with value: 0.9810945173441266 and parameters: {'n_estimators': 50, 'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 2}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 67, n_estimators: 50, max_depth: 10, min_samples_split: 9, min_samples_leaf: 2, ROC-AUC: 0.9810945173441266\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:31,741] Trial 68 finished with value: 0.9810246396393716 and parameters: {'n_estimators': 42, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 68, n_estimators: 42, max_depth: 10, min_samples_split: 6, min_samples_leaf: 3, ROC-AUC: 0.9810246396393716\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:33,357] Trial 69 finished with value: 0.9680610556808112 and parameters: {'n_estimators': 18, 'max_depth': 8, 'min_samples_split': 8, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 69, n_estimators: 18, max_depth: 8, min_samples_split: 8, min_samples_leaf: 4, ROC-AUC: 0.9680610556808112\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:37,281] Trial 70 finished with value: 0.9776675967228685 and parameters: {'n_estimators': 44, 'max_depth': 9, 'min_samples_split': 10, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 70, n_estimators: 44, max_depth: 9, min_samples_split: 10, min_samples_leaf: 3, ROC-AUC: 0.9776675967228685\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:42,763] Trial 71 finished with value: 0.9814481229853514 and parameters: {'n_estimators': 42, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 71, n_estimators: 42, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9814481229853514\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:47,069] Trial 72 finished with value: 0.9823525936174743 and parameters: {'n_estimators': 46, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 72, n_estimators: 46, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9823525936174743\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:48,740] Trial 73 finished with value: 0.9017830047517534 and parameters: {'n_estimators': 46, 'max_depth': 2, 'min_samples_split': 8, 'min_samples_leaf': 2}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 73, n_estimators: 46, max_depth: 2, min_samples_split: 8, min_samples_leaf: 2, ROC-AUC: 0.9017830047517534\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:54,750] Trial 74 finished with value: 0.9836860302176756 and parameters: {'n_estimators': 48, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 74, n_estimators: 48, max_depth: 10, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9836860302176756\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:12:59,417] Trial 75 finished with value: 0.9821141193461864 and parameters: {'n_estimators': 49, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 5}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 75, n_estimators: 49, max_depth: 10, min_samples_split: 7, min_samples_leaf: 5, ROC-AUC: 0.9821141193461864\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:01,669] Trial 76 finished with value: 0.9700827465888052 and parameters: {'n_estimators': 24, 'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 76, n_estimators: 24, max_depth: 9, min_samples_split: 6, min_samples_leaf: 4, ROC-AUC: 0.9700827465888052\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:06,483] Trial 77 finished with value: 0.9797292758362325 and parameters: {'n_estimators': 48, 'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 5}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 77, n_estimators: 48, max_depth: 9, min_samples_split: 7, min_samples_leaf: 5, ROC-AUC: 0.9797292758362325\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:12,916] Trial 78 finished with value: 0.981608349172723 and parameters: {'n_estimators': 40, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 78, n_estimators: 40, max_depth: 10, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.981608349172723\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:16,928] Trial 79 finished with value: 0.9795581129022665 and parameters: {'n_estimators': 44, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 6}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 79, n_estimators: 44, max_depth: 10, min_samples_split: 6, min_samples_leaf: 6, ROC-AUC: 0.9795581129022665\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:21,251] Trial 80 finished with value: 0.9814499473596505 and parameters: {'n_estimators': 49, 'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 80, n_estimators: 49, max_depth: 9, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9814499473596505\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:27,160] Trial 81 finished with value: 0.9797957586321455 and parameters: {'n_estimators': 46, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 81, n_estimators: 46, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9797957586321455\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:31,608] Trial 82 finished with value: 0.9825775030299285 and parameters: {'n_estimators': 47, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 82, n_estimators: 47, max_depth: 10, min_samples_split: 8, min_samples_leaf: 3, ROC-AUC: 0.9825775030299285\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:36,977] Trial 83 finished with value: 0.9796389500461513 and parameters: {'n_estimators': 48, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 2}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 83, n_estimators: 48, max_depth: 10, min_samples_split: 8, min_samples_leaf: 2, ROC-AUC: 0.9796389500461513\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:42,305] Trial 84 finished with value: 0.9834033704135695 and parameters: {'n_estimators': 50, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 3}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 84, n_estimators: 50, max_depth: 10, min_samples_split: 7, min_samples_leaf: 3, ROC-AUC: 0.9834033704135695\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:46,461] Trial 85 finished with value: 0.9790578976503749 and parameters: {'n_estimators': 47, 'max_depth': 9, 'min_samples_split': 7, 'min_samples_leaf': 4}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 85, n_estimators: 47, max_depth: 9, min_samples_split: 7, min_samples_leaf: 4, ROC-AUC: 0.9790578976503749\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:52,883] Trial 86 finished with value: 0.9823975823604989 and parameters: {'n_estimators': 50, 'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 2}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 86, n_estimators: 50, max_depth: 10, min_samples_split: 9, min_samples_leaf: 2, ROC-AUC: 0.9823975823604989\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:13:57,205] Trial 87 finished with value: 0.9828272222441055 and parameters: {'n_estimators': 44, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 87, n_estimators: 44, max_depth: 10, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9828272222441055\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:00,666] Trial 88 finished with value: 0.9776859650109391 and parameters: {'n_estimators': 38, 'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 88, n_estimators: 38, max_depth: 9, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9776859650109391\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:03,969] Trial 89 finished with value: 0.9451653474176507 and parameters: {'n_estimators': 45, 'max_depth': 5, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 89, n_estimators: 45, max_depth: 5, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9451653474176507\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:09,086] Trial 90 finished with value: 0.9838677314595337 and parameters: {'n_estimators': 43, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 90, n_estimators: 43, max_depth: 10, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9838677314595337\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:13,247] Trial 91 finished with value: 0.9841270548935176 and parameters: {'n_estimators': 43, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 91, n_estimators: 43, max_depth: 10, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9841270548935176\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:17,983] Trial 92 finished with value: 0.9839235487638099 and parameters: {'n_estimators': 43, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 92, n_estimators: 43, max_depth: 10, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9839235487638099\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:22,760] Trial 93 finished with value: 0.9831544443826644 and parameters: {'n_estimators': 41, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 93, n_estimators: 41, max_depth: 10, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9831544443826644\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:26,710] Trial 94 finished with value: 0.9839208098275607 and parameters: {'n_estimators': 41, 'max_depth': 10, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 94, n_estimators: 41, max_depth: 10, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9839208098275607\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:30,407] Trial 95 finished with value: 0.9823645583886957 and parameters: {'n_estimators': 37, 'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 95, n_estimators: 37, max_depth: 10, min_samples_split: 5, min_samples_leaf: 1, ROC-AUC: 0.9823645583886957\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:35,277] Trial 96 finished with value: 0.9800027024170991 and parameters: {'n_estimators': 39, 'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 96, n_estimators: 39, max_depth: 9, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9800027024170991\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:39,209] Trial 97 finished with value: 0.9819112464965365 and parameters: {'n_estimators': 41, 'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 97, n_estimators: 41, max_depth: 10, min_samples_split: 5, min_samples_leaf: 1, ROC-AUC: 0.9819112464965365\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:42,012] Trial 98 finished with value: 0.9578476782454327 and parameters: {'n_estimators': 40, 'max_depth': 6, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 98, n_estimators: 40, max_depth: 6, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9578476782454327\n"]},{"output_type":"stream","name":"stderr","text":["[I 2024-05-13 04:14:45,880] Trial 99 finished with value: 0.9797870874449568 and parameters: {'n_estimators': 31, 'max_depth': 9, 'min_samples_split': 6, 'min_samples_leaf': 1}. Best is trial 56 with value: 0.9844000197076754.\n"]},{"output_type":"stream","name":"stdout","text":["Trial 99, n_estimators: 31, max_depth: 9, min_samples_split: 6, min_samples_leaf: 1, ROC-AUC: 0.9797870874449568\n","Best Parameters: {'n_estimators': 48, 'max_depth': 10, 'min_samples_split': 8, 'min_samples_leaf': 4}\n","Best ROC-AUC: Score: 0.9844000197076754\n"]}]},{"cell_type":"code","source":["# XGBoost\n","print('\\nXGBoost started')\n","def objective_function(trial):\n"," n_estimators = trial.suggest_int('n_estimators', 2, 50)\n"," max_depth = trial.suggest_int('max_depth', 1, 10)\n"," learning_rate = trial.suggest_float('learning_rate', 0.001, 0.9, log=True)\n"," min_child_weight = trial.suggest_int('min_child_weight', 1, 10)\n"," subsample = trial.suggest_float('subsample', 0.5, 1.0)\n"," colsample_bytree = trial.suggest_float('colsample_bytree', 0.5, 1.0)\n"," gamma = trial.suggest_float('gamma', 0, 1.0)\n"," reg_alpha = trial.suggest_float('reg_alpha', 0, 1)\n"," reg_lambda = trial.suggest_float('reg_lambda', 0, 1)\n","\n"," model = XGBClassifier(\n"," n_estimators=n_estimators,\n"," max_depth=max_depth,\n"," learning_rate=learning_rate,\n"," min_child_weight=min_child_weight,\n"," subsample=subsample,\n"," colsample_bytree=colsample_bytree,\n"," gamma=gamma,\n"," reg_alpha=reg_alpha,\n"," reg_lambda=reg_lambda,\n"," use_label_encoder=False,\n"," n_jobs=-1\n"," )\n","\n"," # Using cross_val_score to get the average ROC-AUC score for each fold\n"," scores = cross_val_score(model, X_train_oversampled, y_train_oversampled_encoded, cv=5, scoring='roc_auc')\n"," roc_auc = np.mean(scores)\n"," # Printing intermediate results\n"," print(f\"Trial {trial.number}, n_estimators: {n_estimators}, max_depth: {max_depth}, learning_rate: {learning_rate},\"\n"," f\"min_child_weight: {min_child_weight}, subsample: {subsample}, colsample_bytree: {colsample_bytree}, \"\n"," f\"gamma: {gamma}, reg_alpha: {reg_alpha}, reg_lambda: {reg_lambda}, ROC-AUC: {roc_auc}\")\n"," return roc_auc\n","\n","\n","study_xgb = optuna.create_study(direction=\"maximize\")\n","study_xgb.optimize(objective_function, n_trials=100)\n","\n","best_params_xgb = study_xgb.best_params\n","print(\"Best Parameters: \", best_params_xgb)\n","print(\"Best ROC-AUC Score: \", study_xgb.best_value)\n","\n","best_model_xgb = XGBClassifier(**best_params_xgb, use_label_encoder=False, n_jobs=-1)\n","with open('storage_files/best_models_xgb_36_features.pkl', 'wb') as file:\n"," pickle.dump(best_model_xgb, file)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rzboVwPUDyd-","executionInfo":{"status":"ok","timestamp":1715573885580,"user_tz":-480,"elapsed":199195,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"2b611c3d-0a20-4a34-f30b-0de153f1dbae"},"execution_count":10,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","XGBoost started\n","Trial 0, n_estimators: 3, max_depth: 2, learning_rate: 0.0030813089476601644,min_child_weight: 7, subsample: 0.5311549574061932, colsample_bytree: 0.9904951786289279, gamma: 0.16263061376276244, reg_alpha: 0.17079289788979046, reg_lambda: 0.044208855112493506, ROC-AUC: 0.7811500536303772\n","Trial 1, n_estimators: 42, max_depth: 6, learning_rate: 0.23544204420936748,min_child_weight: 4, subsample: 0.8129557133097995, colsample_bytree: 0.9346641825707578, gamma: 0.9414249643904902, reg_alpha: 0.09842774179324987, reg_lambda: 0.6936135940781807, ROC-AUC: 0.9875758294818295\n","Trial 2, n_estimators: 18, max_depth: 6, learning_rate: 0.03898530000013839,min_child_weight: 2, subsample: 0.666965501471807, colsample_bytree: 0.9470189253050632, gamma: 0.4301248503188094, reg_alpha: 0.15748589606396823, reg_lambda: 0.041994804639860606, ROC-AUC: 0.9483920401889658\n","Trial 3, n_estimators: 41, max_depth: 6, learning_rate: 0.0018380184708672758,min_child_weight: 10, subsample: 0.7601484948105508, colsample_bytree: 0.7277397791914144, gamma: 0.4815664719739068, reg_alpha: 0.22722362932545204, reg_lambda: 0.12610145496659053, ROC-AUC: 0.9440163520510232\n","Trial 4, n_estimators: 48, max_depth: 2, learning_rate: 0.06933210310329908,min_child_weight: 1, subsample: 0.6941930539930846, colsample_bytree: 0.7112950908115747, gamma: 0.8009062546979172, reg_alpha: 0.13068890096957775, reg_lambda: 0.5979283883982579, ROC-AUC: 0.9268620724056109\n","Trial 5, n_estimators: 22, max_depth: 6, learning_rate: 0.013734934952490313,min_child_weight: 6, subsample: 0.6741180561197793, colsample_bytree: 0.6850127070851553, gamma: 0.9820401546195002, reg_alpha: 0.6936155082345913, reg_lambda: 0.5016818509762447, ROC-AUC: 0.9459791239017923\n","Trial 6, n_estimators: 29, max_depth: 7, learning_rate: 0.001715248030805128,min_child_weight: 10, subsample: 0.8286462777251258, colsample_bytree: 0.6535622783956236, gamma: 0.9133612650797344, reg_alpha: 0.6561116984432649, reg_lambda: 0.6772410309965773, ROC-AUC: 0.9540599096338911\n","Trial 7, n_estimators: 12, max_depth: 9, learning_rate: 0.4371260229270629,min_child_weight: 5, subsample: 0.8548791380573488, colsample_bytree: 0.5359282141314967, gamma: 0.31961167410782554, reg_alpha: 0.26811189956471526, reg_lambda: 0.8405942332690637, ROC-AUC: 0.9810041730833768\n","Trial 8, n_estimators: 42, max_depth: 2, learning_rate: 0.002321377225708867,min_child_weight: 8, subsample: 0.552339685704462, colsample_bytree: 0.5927175564032442, gamma: 0.26226562528446706, reg_alpha: 0.09238354873628463, reg_lambda: 0.4505428735719178, ROC-AUC: 0.8803770363700758\n","Trial 9, n_estimators: 5, max_depth: 2, learning_rate: 0.10114125978035127,min_child_weight: 6, subsample: 0.8946457109980646, colsample_bytree: 0.556409231992187, gamma: 0.8296527917199729, reg_alpha: 0.7682700010639031, reg_lambda: 0.223419845675084, ROC-AUC: 0.8620566282299448\n","Trial 10, n_estimators: 32, max_depth: 10, learning_rate: 0.8535653096783381,min_child_weight: 3, subsample: 0.980793825532664, colsample_bytree: 0.8612972591053797, gamma: 0.6517576127176568, reg_alpha: 0.43472424767974727, reg_lambda: 0.8503374817724212, ROC-AUC: 0.9773894902023764\n","Trial 11, n_estimators: 11, max_depth: 9, learning_rate: 0.5151986813570824,min_child_weight: 4, subsample: 0.8698407184709971, colsample_bytree: 0.8335788941560807, gamma: 0.03664824166041836, reg_alpha: 0.359986939435709, reg_lambda: 0.983542523702137, ROC-AUC: 0.9810504917145225\n","Trial 12, n_estimators: 35, max_depth: 8, learning_rate: 0.15758914068732763,min_child_weight: 4, subsample: 0.960567974464, colsample_bytree: 0.8468210088108845, gamma: 0.0036013965046293173, reg_alpha: 0.9793976823051895, reg_lambda: 0.9952944318380443, ROC-AUC: 0.986266482855019\n","Trial 13, n_estimators: 36, max_depth: 4, learning_rate: 0.18186534184316291,min_child_weight: 4, subsample: 0.9983932496440248, colsample_bytree: 0.84356020689548, gamma: 0.6563204211056423, reg_alpha: 0.9955326988153764, reg_lambda: 0.9862501842836836, ROC-AUC: 0.9761015611071138\n","Trial 14, n_estimators: 46, max_depth: 8, learning_rate: 0.21167493041691612,min_child_weight: 3, subsample: 0.9338214867138319, colsample_bytree: 0.9102829385627371, gamma: 0.6507675714224619, reg_alpha: 0.9517178526058757, reg_lambda: 0.765152146232969, ROC-AUC: 0.9889923893669239\n","Trial 15, n_estimators: 48, max_depth: 4, learning_rate: 0.024527079805075878,min_child_weight: 1, subsample: 0.7815330260290169, colsample_bytree: 0.9049153067123268, gamma: 0.637949715832308, reg_alpha: 0.5471841675246776, reg_lambda: 0.7179246558242417, ROC-AUC: 0.9589494164070957\n","Trial 16, n_estimators: 42, max_depth: 4, learning_rate: 0.27374784651739875,min_child_weight: 3, subsample: 0.9112043655215127, colsample_bytree: 0.7913383762879052, gamma: 0.749488478064928, reg_alpha: 0.020429658611381518, reg_lambda: 0.31740891262308435, ROC-AUC: 0.9808835411484061\n","Trial 17, n_estimators: 47, max_depth: 8, learning_rate: 0.009054124331435574,min_child_weight: 2, subsample: 0.8048301122526793, colsample_bytree: 0.9794735643945247, gamma: 0.9877252776778052, reg_alpha: 0.8358595382872772, reg_lambda: 0.7913879378879831, ROC-AUC: 0.9532469350406225\n","Trial 18, n_estimators: 39, max_depth: 5, learning_rate: 0.05969378224447004,min_child_weight: 5, subsample: 0.9332933894458514, colsample_bytree: 0.9073599919027312, gamma: 0.6040292492447806, reg_alpha: 0.5188558276809667, reg_lambda: 0.5622865747807393, ROC-AUC: 0.9717179031396462\n","Trial 19, n_estimators: 25, max_depth: 7, learning_rate: 0.24469558657007684,min_child_weight: 8, subsample: 0.6105577462991719, colsample_bytree: 0.9160876825868774, gamma: 0.8754373628145132, reg_alpha: 0.8681403890646294, reg_lambda: 0.4083907100205918, ROC-AUC: 0.9850999266323944\n","Trial 20, n_estimators: 50, max_depth: 10, learning_rate: 0.11347112522941191,min_child_weight: 3, subsample: 0.7212638524380395, colsample_bytree: 0.7757268522550375, gamma: 0.5287599508142377, reg_alpha: 0.36375273510399564, reg_lambda: 0.662025421849189, ROC-AUC: 0.9879372091634948\n","Trial 21, n_estimators: 49, max_depth: 10, learning_rate: 0.09953318734507918,min_child_weight: 3, subsample: 0.7235819082172931, colsample_bytree: 0.7677782829093096, gamma: 0.5339154656418247, reg_alpha: 0.355175956523888, reg_lambda: 0.6995140100316001, ROC-AUC: 0.9873279578622205\n","Trial 22, n_estimators: 44, max_depth: 9, learning_rate: 0.39688560144916685,min_child_weight: 2, subsample: 0.7259857005769577, colsample_bytree: 0.8037753713496629, gamma: 0.37652567218664945, reg_alpha: 0.0022852164130691133, reg_lambda: 0.6341293729945501, ROC-AUC: 0.9872851993204653\n","Trial 23, n_estimators: 50, max_depth: 8, learning_rate: 0.1354969048299756,min_child_weight: 4, subsample: 0.6329139409771672, colsample_bytree: 0.9505316096257062, gamma: 0.7449965522913247, reg_alpha: 0.30988123515013977, reg_lambda: 0.7757920454897534, ROC-AUC: 0.9857275187796034\n","Trial 24, n_estimators: 45, max_depth: 10, learning_rate: 0.7980373512155522,min_child_weight: 3, subsample: 0.8152158821338268, colsample_bytree: 0.8753169982737573, gamma: 0.7334466250381801, reg_alpha: 0.4395792766150708, reg_lambda: 0.8885071302342523, ROC-AUC: 0.9756412773292537\n","Trial 25, n_estimators: 38, max_depth: 7, learning_rate: 0.0398303077008447,min_child_weight: 5, subsample: 0.746697647988653, colsample_bytree: 0.6404074893620489, gamma: 0.5528556910305745, reg_alpha: 0.6041329872081729, reg_lambda: 0.577037547903867, ROC-AUC: 0.9745333253708921\n","Trial 26, n_estimators: 32, max_depth: 9, learning_rate: 0.26650526501020844,min_child_weight: 2, subsample: 0.8555748841902416, colsample_bytree: 0.7603967976315965, gamma: 0.47871442226949756, reg_alpha: 0.4416738086456252, reg_lambda: 0.7694618927018833, ROC-AUC: 0.9885896005577592\n","Trial 27, n_estimators: 31, max_depth: 9, learning_rate: 0.021004857027611294,min_child_weight: 1, subsample: 0.9284649197886767, colsample_bytree: 0.7642570492554345, gamma: 0.45475127284976474, reg_alpha: 0.44418281841530005, reg_lambda: 0.9077572548329713, ROC-AUC: 0.9668825790165693\n","Trial 28, n_estimators: 27, max_depth: 10, learning_rate: 0.08814679160529716,min_child_weight: 2, subsample: 0.8707077324192987, colsample_bytree: 0.7354585643525741, gamma: 0.3833468162081789, reg_alpha: 0.579536580208554, reg_lambda: 0.7696696146537662, ROC-AUC: 0.9840060282139775\n","Trial 29, n_estimators: 21, max_depth: 8, learning_rate: 0.004446233816426357,min_child_weight: 2, subsample: 0.9606323097699669, colsample_bytree: 0.9997279724532719, gamma: 0.2921502470692484, reg_alpha: 0.37208693859226116, reg_lambda: 0.5073063511218491, ROC-AUC: 0.9280629450190897\n","Trial 30, n_estimators: 34, max_depth: 9, learning_rate: 0.36457410246705735,min_child_weight: 1, subsample: 0.5285735586717104, colsample_bytree: 0.815589434811253, gamma: 0.170715035709335, reg_alpha: 0.47734379524282805, reg_lambda: 0.37702120909972814, ROC-AUC: 0.9872904138540062\n","Trial 31, n_estimators: 45, max_depth: 5, learning_rate: 0.22720762031142977,min_child_weight: 3, subsample: 0.8436795457533114, colsample_bytree: 0.9458509408183173, gamma: 0.561868234377066, reg_alpha: 0.2390258033513448, reg_lambda: 0.6556185420134326, ROC-AUC: 0.9834491898089365\n","Trial 32, n_estimators: 40, max_depth: 7, learning_rate: 0.14531668483145616,min_child_weight: 4, subsample: 0.7893207257908467, colsample_bytree: 0.6894104301332091, gamma: 0.7003305506913365, reg_alpha: 0.18845237672533566, reg_lambda: 0.7356888112525266, ROC-AUC: 0.9845324828973222\n","Trial 33, n_estimators: 44, max_depth: 8, learning_rate: 0.5799212096514632,min_child_weight: 3, subsample: 0.8796554890421505, colsample_bytree: 0.8758715974557593, gamma: 0.4088269126407391, reg_alpha: 0.07416316020932297, reg_lambda: 0.8348854666192745, ROC-AUC: 0.9854772322520414\n","Trial 34, n_estimators: 46, max_depth: 10, learning_rate: 0.056372843996192044,min_child_weight: 2, subsample: 0.7668147112325432, colsample_bytree: 0.9595850271593812, gamma: 0.48242503102910234, reg_alpha: 0.30560177427488844, reg_lambda: 0.6330614751528814, ROC-AUC: 0.9816488057414692\n","Trial 35, n_estimators: 50, max_depth: 6, learning_rate: 0.2736137180795617,min_child_weight: 5, subsample: 0.6910759468940925, colsample_bytree: 0.7819575715947145, gamma: 0.5167837700715728, reg_alpha: 0.6835554081601707, reg_lambda: 0.5388079033549927, ROC-AUC: 0.987408310019128\n","Trial 36, n_estimators: 37, max_depth: 9, learning_rate: 0.0391608966791932,min_child_weight: 2, subsample: 0.7342420650464313, colsample_bytree: 0.7115323692656997, gamma: 0.9231210097819967, reg_alpha: 0.18120436330636006, reg_lambda: 0.9086157446117524, ROC-AUC: 0.9794360207290296\n","Trial 37, n_estimators: 42, max_depth: 1, learning_rate: 0.12449393239773421,min_child_weight: 6, subsample: 0.8427736591069742, colsample_bytree: 0.749582550784803, gamma: 0.6031240073721784, reg_alpha: 0.625928801543405, reg_lambda: 0.7615357149110344, ROC-AUC: 0.8904798175123299\n","Trial 38, n_estimators: 31, max_depth: 6, learning_rate: 0.07407872680467407,min_child_weight: 7, subsample: 0.6558557768771203, colsample_bytree: 0.6466584864354968, gamma: 0.7960800638156673, reg_alpha: 0.7331412417986609, reg_lambda: 0.6932685251056893, ROC-AUC: 0.9794520904743574\n","Trial 39, n_estimators: 17, max_depth: 7, learning_rate: 0.35134297127694086,min_child_weight: 4, subsample: 0.9038662406998255, colsample_bytree: 0.9326739573406914, gamma: 0.22664929637350328, reg_alpha: 0.918513050489423, reg_lambda: 0.005331294251474272, ROC-AUC: 0.9822375181872756\n","Trial 40, n_estimators: 42, max_depth: 8, learning_rate: 0.18018258169037166,min_child_weight: 3, subsample: 0.5817059712507886, colsample_bytree: 0.8825571209453613, gamma: 0.34521773762839875, reg_alpha: 0.38634669516878983, reg_lambda: 0.6445366990956899, ROC-AUC: 0.9850901577597726\n","Trial 41, n_estimators: 50, max_depth: 6, learning_rate: 0.2919778470190327,min_child_weight: 5, subsample: 0.6807564243802677, colsample_bytree: 0.7989041228371506, gamma: 0.49301939095177216, reg_alpha: 0.6836829950432646, reg_lambda: 0.5614563709435013, ROC-AUC: 0.986337756942301\n","Trial 42, n_estimators: 47, max_depth: 5, learning_rate: 0.0010969189421092328,min_child_weight: 5, subsample: 0.7176441113823822, colsample_bytree: 0.7775087159772366, gamma: 0.5148879991290229, reg_alpha: 0.7772507865460739, reg_lambda: 0.5131391973657291, ROC-AUC: 0.9314639161983876\n","Trial 43, n_estimators: 48, max_depth: 3, learning_rate: 0.6895672675111988,min_child_weight: 7, subsample: 0.6940565616729795, colsample_bytree: 0.680328866705258, gamma: 0.45543076718255854, reg_alpha: 0.1249629166611066, reg_lambda: 0.813646048016436, ROC-AUC: 0.9798586322594636\n","Trial 44, n_estimators: 50, max_depth: 7, learning_rate: 0.2118686329957139,min_child_weight: 4, subsample: 0.7612331464372117, colsample_bytree: 0.6082924653270734, gamma: 0.5798222438540268, reg_alpha: 0.9310602035424198, reg_lambda: 0.6009621444209368, ROC-AUC: 0.9886233438301615\n","Trial 45, n_estimators: 44, max_depth: 7, learning_rate: 0.20038795001626097,min_child_weight: 4, subsample: 0.8183928663341457, colsample_bytree: 0.50272725923232, gamma: 0.6746541478457839, reg_alpha: 0.9416485259095149, reg_lambda: 0.6031389619398304, ROC-AUC: 0.9877172509456242\n","Trial 46, n_estimators: 44, max_depth: 9, learning_rate: 0.5151435296888542,min_child_weight: 4, subsample: 0.828467374246892, colsample_bytree: 0.5942994334941812, gamma: 0.6858972833297673, reg_alpha: 0.9105387050566294, reg_lambda: 0.6059120967574347, ROC-AUC: 0.9864257781207059\n","Trial 47, n_estimators: 34, max_depth: 7, learning_rate: 0.181844158189506,min_child_weight: 3, subsample: 0.7619419651663925, colsample_bytree: 0.500147105516384, gamma: 0.5896350047229881, reg_alpha: 0.9239731089829599, reg_lambda: 0.44727059351773246, ROC-AUC: 0.9866505592939484\n","Trial 48, n_estimators: 40, max_depth: 8, learning_rate: 0.12011009794902958,min_child_weight: 1, subsample: 0.7934137667584411, colsample_bytree: 0.60804742189835, gamma: 0.6402013010180998, reg_alpha: 0.8271091624526938, reg_lambda: 0.7327351814234224, ROC-AUC: 0.9859130307842081\n","Trial 49, n_estimators: 4, max_depth: 9, learning_rate: 0.22418051292384775,min_child_weight: 6, subsample: 0.8240960678771657, colsample_bytree: 0.5431475505046498, gamma: 0.8132269331589079, reg_alpha: 0.970016278459312, reg_lambda: 0.600801216609358, ROC-AUC: 0.965797719351641\n","Trial 50, n_estimators: 47, max_depth: 7, learning_rate: 0.4449972884731887,min_child_weight: 4, subsample: 0.8598452295200737, colsample_bytree: 0.5027285468984571, gamma: 0.6934404231946967, reg_alpha: 0.8743236645998179, reg_lambda: 0.11011727450049713, ROC-AUC: 0.986964727419587\n","Trial 51, n_estimators: 43, max_depth: 6, learning_rate: 0.2039619688437467,min_child_weight: 4, subsample: 0.7612828666372264, colsample_bytree: 0.5728454484353368, gamma: 0.8743513310197656, reg_alpha: 0.9623731733233485, reg_lambda: 0.6921893499476592, ROC-AUC: 0.9861116110505345\n","Trial 52, n_estimators: 46, max_depth: 7, learning_rate: 0.3281684850239028,min_child_weight: 3, subsample: 0.7849928827151296, colsample_bytree: 0.5257451601127898, gamma: 0.5932618616924673, reg_alpha: 0.8115830011448047, reg_lambda: 0.6781763269315846, ROC-AUC: 0.9866101750246757\n","Trial 53, n_estimators: 48, max_depth: 8, learning_rate: 0.08150938257356502,min_child_weight: 10, subsample: 0.8882884349116107, colsample_bytree: 0.6250745590510266, gamma: 0.432592467332222, reg_alpha: 0.5060563913251793, reg_lambda: 0.8623825037626289, ROC-AUC: 0.986735962914782\n","Trial 54, n_estimators: 40, max_depth: 10, learning_rate: 0.11169464575984375,min_child_weight: 2, subsample: 0.8066292075308629, colsample_bytree: 0.6747546692744568, gamma: 0.5602019130856541, reg_alpha: 0.8816633067558357, reg_lambda: 0.46901373995724305, ROC-AUC: 0.9872909711404567\n","Trial 55, n_estimators: 38, max_depth: 6, learning_rate: 0.056676085267877985,min_child_weight: 3, subsample: 0.9292537428041471, colsample_bytree: 0.8281957498015702, gamma: 0.9462105364299983, reg_alpha: 0.9430624992637344, reg_lambda: 0.7360576999614223, ROC-AUC: 0.9744484793003758\n","Trial 56, n_estimators: 50, max_depth: 8, learning_rate: 0.16377414465567544,min_child_weight: 4, subsample: 0.7451315429723853, colsample_bytree: 0.9776906125130544, gamma: 0.7609180453510829, reg_alpha: 0.05717612835606205, reg_lambda: 0.8117634357488934, ROC-AUC: 0.9865797807483115\n","Trial 57, n_estimators: 28, max_depth: 5, learning_rate: 0.6296277193800303,min_child_weight: 2, subsample: 0.8454419734498714, colsample_bytree: 0.714461915986999, gamma: 0.869414372923742, reg_alpha: 0.9938514563082529, reg_lambda: 0.6630353615395497, ROC-AUC: 0.9804286708595716\n","Trial 58, n_estimators: 46, max_depth: 7, learning_rate: 0.27518949124814285,min_child_weight: 3, subsample: 0.7078180825253857, colsample_bytree: 0.7330066640936537, gamma: 0.6560120189607843, reg_alpha: 0.2906170708054786, reg_lambda: 0.9523834742151955, ROC-AUC: 0.9884366449531822\n","Trial 59, n_estimators: 23, max_depth: 9, learning_rate: 0.4047693785570528,min_child_weight: 3, subsample: 0.710111722922886, colsample_bytree: 0.5747338991019817, gamma: 0.6698817427802759, reg_alpha: 0.2988590407286591, reg_lambda: 0.9456066987578371, ROC-AUC: 0.9867514714156591\n","Trial 60, n_estimators: 9, max_depth: 7, learning_rate: 0.2867569503501246,min_child_weight: 4, subsample: 0.9582676646175503, colsample_bytree: 0.7309213618395936, gamma: 0.6173364082014843, reg_alpha: 0.239881390667184, reg_lambda: 0.9345496692650958, ROC-AUC: 0.9743288022989585\n","Trial 61, n_estimators: 46, max_depth: 7, learning_rate: 0.2375571929059465,min_child_weight: 3, subsample: 0.7795487690938865, colsample_bytree: 0.9187180654338607, gamma: 0.721351847301981, reg_alpha: 0.3938337574730741, reg_lambda: 0.6137077960679707, ROC-AUC: 0.9894461931065945\n","Trial 62, n_estimators: 48, max_depth: 7, learning_rate: 0.2202503184488217,min_child_weight: 3, subsample: 0.6517169183227667, colsample_bytree: 0.8972329067985417, gamma: 0.7042936461410385, reg_alpha: 0.41896084858384425, reg_lambda: 0.6132506732553328, ROC-AUC: 0.9873886023437357\n","Trial 63, n_estimators: 46, max_depth: 8, learning_rate: 0.14204821715798485,min_child_weight: 3, subsample: 0.7059104327493406, colsample_bytree: 0.7504198403933782, gamma: 0.6406304172542144, reg_alpha: 0.38735350824127657, reg_lambda: 0.5711259206523444, ROC-AUC: 0.9865305025883633\n","Trial 64, n_estimators: 44, max_depth: 10, learning_rate: 0.4499583110799482,min_child_weight: 2, subsample: 0.7731325759968114, colsample_bytree: 0.8495436434893244, gamma: 0.7827016157553471, reg_alpha: 0.3571520824606558, reg_lambda: 0.5436971034844151, ROC-AUC: 0.9838242298691775\n","Trial 65, n_estimators: 49, max_depth: 8, learning_rate: 0.2495204648245933,min_child_weight: 2, subsample: 0.7431019304756589, colsample_bytree: 0.9232047355101101, gamma: 0.7267156828135091, reg_alpha: 0.47606653016218436, reg_lambda: 0.4781253339652878, ROC-AUC: 0.9871146591118889\n","Trial 66, n_estimators: 45, max_depth: 7, learning_rate: 0.10207427488686166,min_child_weight: 3, subsample: 0.8033928878760276, colsample_bytree: 0.6605274206538354, gamma: 0.5393789790833993, reg_alpha: 0.5498344057380526, reg_lambda: 0.7128946466613137, ROC-AUC: 0.9856163844622964\n","Trial 67, n_estimators: 41, max_depth: 9, learning_rate: 0.15963270251526018,min_child_weight: 1, subsample: 0.9888125720325525, colsample_bytree: 0.8187093215655059, gamma: 0.5728257551614727, reg_alpha: 0.269830294623746, reg_lambda: 0.8612161666047793, ROC-AUC: 0.987896529891262\n","Trial 68, n_estimators: 41, max_depth: 9, learning_rate: 0.007796809676041261,min_child_weight: 1, subsample: 0.98076066681438, colsample_bytree: 0.8126783798760888, gamma: 0.5757190081904777, reg_alpha: 0.2790508202036535, reg_lambda: 0.9653196674943099, ROC-AUC: 0.9627804744601741\n","Trial 69, n_estimators: 36, max_depth: 10, learning_rate: 0.15994134914662222,min_child_weight: 1, subsample: 0.9967612237982746, colsample_bytree: 0.7480406048795929, gamma: 0.45429788339704935, reg_alpha: 0.20141167032195495, reg_lambda: 0.8776853154266862, ROC-AUC: 0.9899778459637453\n","Trial 70, n_estimators: 36, max_depth: 10, learning_rate: 0.3181891677135062,min_child_weight: 1, subsample: 0.9542390224348631, colsample_bytree: 0.6934793421499452, gamma: 0.4076018888833913, reg_alpha: 0.33880765101845434, reg_lambda: 0.8953737943038744, ROC-AUC: 0.9880832387951755\n","Trial 71, n_estimators: 32, max_depth: 10, learning_rate: 0.3132881092927488,min_child_weight: 1, subsample: 0.9494594315607459, colsample_bytree: 0.7552065090417123, gamma: 0.4517239323449611, reg_alpha: 0.3341488055300729, reg_lambda: 0.8951252221615495, ROC-AUC: 0.9870859947455488\n","Trial 72, n_estimators: 36, max_depth: 10, learning_rate: 0.5319672354353969,min_child_weight: 1, subsample: 0.9754090005333209, colsample_bytree: 0.7240420986778209, gamma: 0.38685587316847303, reg_alpha: 0.4040452730442723, reg_lambda: 0.8033570268184258, ROC-AUC: 0.9848382679781981\n","Trial 73, n_estimators: 34, max_depth: 10, learning_rate: 0.25042175674265155,min_child_weight: 2, subsample: 0.9971838608764589, colsample_bytree: 0.6993730904856159, gamma: 0.42033227455436317, reg_alpha: 0.21015034585680087, reg_lambda: 0.8481948237855086, ROC-AUC: 0.9894457698644228\n","Trial 74, n_estimators: 32, max_depth: 10, learning_rate: 0.3598665655944454,min_child_weight: 2, subsample: 0.9970187410720095, colsample_bytree: 0.6857792085623985, gamma: 0.35818541739551074, reg_alpha: 0.12403237188341631, reg_lambda: 0.9334770098588876, ROC-AUC: 0.9886449983139973\n","Trial 75, n_estimators: 33, max_depth: 10, learning_rate: 0.385848239419273,min_child_weight: 2, subsample: 0.9985245594808541, colsample_bytree: 0.6645394965281587, gamma: 0.343133017762825, reg_alpha: 0.19391517065769986, reg_lambda: 0.9356359221340627, ROC-AUC: 0.9885896644134984\n","Trial 76, n_estimators: 30, max_depth: 10, learning_rate: 0.429768296856752,min_child_weight: 2, subsample: 0.9906518275649104, colsample_bytree: 0.6993336284018296, gamma: 0.3239851739173102, reg_alpha: 0.15254413112144416, reg_lambda: 0.8404463636122679, ROC-AUC: 0.9880077396744426\n","Trial 77, n_estimators: 34, max_depth: 10, learning_rate: 0.8027915378793494,min_child_weight: 2, subsample: 0.9706678418214009, colsample_bytree: 0.6658738535737136, gamma: 0.279726440102487, reg_alpha: 0.19197111242555237, reg_lambda: 0.9787163233995227, ROC-AUC: 0.9840845058619145\n","Trial 78, n_estimators: 25, max_depth: 9, learning_rate: 0.3702108130729248,min_child_weight: 2, subsample: 0.9962242329920694, colsample_bytree: 0.6342910551082734, gamma: 0.3503869500201024, reg_alpha: 0.2203415093824134, reg_lambda: 0.9321039435875561, ROC-AUC: 0.988630912054583\n","Trial 79, n_estimators: 24, max_depth: 9, learning_rate: 0.6451895926643367,min_child_weight: 2, subsample: 0.9940899221328094, colsample_bytree: 0.6279214685981281, gamma: 0.2331999698207512, reg_alpha: 0.1098866621790503, reg_lambda: 0.9206870045265203, ROC-AUC: 0.9867348435922801\n","Trial 80, n_estimators: 20, max_depth: 10, learning_rate: 0.3731609472305762,min_child_weight: 1, subsample: 0.9396307159222211, colsample_bytree: 0.6379933036060803, gamma: 0.32568193300313036, reg_alpha: 0.20361458980874125, reg_lambda: 0.9944576141876172, ROC-AUC: 0.9880113166513012\n","Trial 81, n_estimators: 26, max_depth: 9, learning_rate: 0.23680598677598627,min_child_weight: 2, subsample: 0.9995790639833232, colsample_bytree: 0.6145491170767166, gamma: 0.36540823507918657, reg_alpha: 0.22185616236728814, reg_lambda: 0.8561939861052079, ROC-AUC: 0.9869592732950908\n","Trial 82, n_estimators: 33, max_depth: 9, learning_rate: 0.5295600430982147,min_child_weight: 2, subsample: 0.9167199780862598, colsample_bytree: 0.7033091936196489, gamma: 0.3531477883959998, reg_alpha: 0.04101117266083787, reg_lambda: 0.7812302835733552, ROC-AUC: 0.9869990773215107\n","Trial 83, n_estimators: 30, max_depth: 10, learning_rate: 0.18578194081534,min_child_weight: 2, subsample: 0.9689340146227441, colsample_bytree: 0.6542144065267533, gamma: 0.4035347622955036, reg_alpha: 0.22645844899228817, reg_lambda: 0.8326436815795283, ROC-AUC: 0.9889236320401373\n","Trial 84, n_estimators: 29, max_depth: 10, learning_rate: 0.17495096857931788,min_child_weight: 9, subsample: 0.9640340295591305, colsample_bytree: 0.6664563500815468, gamma: 0.40939017203941047, reg_alpha: 0.15221332947862753, reg_lambda: 0.8768903509017032, ROC-AUC: 0.9874823277922596\n","Trial 85, n_estimators: 28, max_depth: 10, learning_rate: 0.14347473700825067,min_child_weight: 2, subsample: 0.9467896881244763, colsample_bytree: 0.6484344337648664, gamma: 0.29319331962079614, reg_alpha: 0.247783194061793, reg_lambda: 0.927363372402986, ROC-AUC: 0.9876330167819637\n","Trial 86, n_estimators: 30, max_depth: 10, learning_rate: 0.1970316612728669,min_child_weight: 1, subsample: 0.5096416565307481, colsample_bytree: 0.5992119197091593, gamma: 0.2497268276801594, reg_alpha: 0.09212378093836682, reg_lambda: 0.8398485811081274, ROC-AUC: 0.9894794756677548\n","Trial 87, n_estimators: 30, max_depth: 10, learning_rate: 0.19245399909026265,min_child_weight: 1, subsample: 0.502815777613247, colsample_bytree: 0.5934526108555667, gamma: 0.1832844420089611, reg_alpha: 0.10176815631869439, reg_lambda: 0.8280492874901987, ROC-AUC: 0.9885207128808278\n","Trial 88, n_estimators: 26, max_depth: 9, learning_rate: 0.01727935063182047,min_child_weight: 1, subsample: 0.9792565781966783, colsample_bytree: 0.609380888373792, gamma: 0.06937418936816309, reg_alpha: 0.07983706880841307, reg_lambda: 0.7548601306168992, ROC-AUC: 0.9711726600924596\n","Trial 89, n_estimators: 31, max_depth: 9, learning_rate: 0.09096955224837784,min_child_weight: 1, subsample: 0.6015987708292678, colsample_bytree: 0.5674578969784033, gamma: 0.25265661837414227, reg_alpha: 0.13590168630283606, reg_lambda: 0.8714844685555876, ROC-AUC: 0.9856800628830211\n","Trial 90, n_estimators: 27, max_depth: 10, learning_rate: 0.253558172805892,min_child_weight: 2, subsample: 0.9675838609070249, colsample_bytree: 0.6202236399070934, gamma: 0.4650250154546003, reg_alpha: 0.17100819823246438, reg_lambda: 0.22934309460676244, ROC-AUC: 0.9875595341304809\n","Trial 91, n_estimators: 33, max_depth: 10, learning_rate: 0.3526587652185769,min_child_weight: 2, subsample: 0.9888069678484074, colsample_bytree: 0.6012407525417319, gamma: 0.3960665249694216, reg_alpha: 0.2157634977604579, reg_lambda: 0.9122687486730986, ROC-AUC: 0.9888833907866111\n","Trial 92, n_estimators: 29, max_depth: 10, learning_rate: 0.21377401805809298,min_child_weight: 2, subsample: 0.9831345580933895, colsample_bytree: 0.6315287174536275, gamma: 0.42366441445619896, reg_alpha: 0.22783686171941786, reg_lambda: 0.8381639996730322, ROC-AUC: 0.988727591226842\n","Trial 93, n_estimators: 29, max_depth: 10, learning_rate: 0.33820551587708736,min_child_weight: 1, subsample: 0.9868578988581144, colsample_bytree: 0.6346810361012609, gamma: 0.42460552198444373, reg_alpha: 0.21685320049684645, reg_lambda: 0.8224659036266525, ROC-AUC: 0.9897505364198729\n","Trial 94, n_estimators: 35, max_depth: 10, learning_rate: 0.31500785610276,min_child_weight: 1, subsample: 0.9845683326517206, colsample_bytree: 0.648798711526616, gamma: 0.43119465345887686, reg_alpha: 0.16305787066500838, reg_lambda: 0.8276585805761189, ROC-AUC: 0.9878707047305133\n","Trial 95, n_estimators: 29, max_depth: 10, learning_rate: 0.125106953496503,min_child_weight: 1, subsample: 0.9202676696080926, colsample_bytree: 0.6821780737217014, gamma: 0.39585711071160146, reg_alpha: 0.25191818068525684, reg_lambda: 0.7909376961086199, ROC-AUC: 0.9872729690993529\n","Trial 96, n_estimators: 30, max_depth: 10, learning_rate: 0.06814595184147024,min_child_weight: 1, subsample: 0.9370950705069797, colsample_bytree: 0.6554728882788003, gamma: 0.5062100527036504, reg_alpha: 0.13658283764587015, reg_lambda: 0.8914924156614094, ROC-AUC: 0.9814679657604893\n","Trial 97, n_estimators: 32, max_depth: 10, learning_rate: 0.46036526938439815,min_child_weight: 1, subsample: 0.9717846698718999, colsample_bytree: 0.5841692631535633, gamma: 0.4221527105676825, reg_alpha: 0.21541600387760257, reg_lambda: 0.8455828164624253, ROC-AUC: 0.9888812254965476\n","Trial 98, n_estimators: 38, max_depth: 10, learning_rate: 0.03065272418500876,min_child_weight: 1, subsample: 0.9700831095318805, colsample_bytree: 0.5789153259754029, gamma: 0.43071258869657014, reg_alpha: 0.25995647653567305, reg_lambda: 0.7531220674773847, ROC-AUC: 0.9842927795349891\n","Trial 99, n_estimators: 28, max_depth: 10, learning_rate: 0.8983475747331252,min_child_weight: 1, subsample: 0.9460716394871967, colsample_bytree: 0.5968337564838441, gamma: 0.4527520019659065, reg_alpha: 0.3199856905149667, reg_lambda: 0.816020408065151, ROC-AUC: 0.9820502166387749\n","Best Parameters: {'n_estimators': 36, 'max_depth': 10, 'learning_rate': 0.15994134914662222, 'min_child_weight': 1, 'subsample': 0.9967612237982746, 'colsample_bytree': 0.7480406048795929, 'gamma': 0.45429788339704935, 'reg_alpha': 0.20141167032195495, 'reg_lambda': 0.8776853154266862}\n","Best ROC-AUC Score: 0.9899778459637453\n"]}]},{"cell_type":"code","source":["# Model selection - Compare Performance\n","with open('storage_files/best_models_lr_36_features.pkl', 'rb') as file:\n"," best_model_lr = pickle.load(file)\n","with open('storage_files/best_models_dt_36_features.pkl', 'rb') as file:\n"," best_model_dt = pickle.load(file)\n","with open('storage_files/best_models_knn_36_features.pkl', 'rb') as file:\n"," best_model_knn = pickle.load(file)\n","with open('storage_files/best_models_rf_36_features.pkl', 'rb') as file:\n"," best_model_rf = pickle.load(file)\n","with open('storage_files/best_models_xgb_36_features.pkl', 'rb') as file:\n"," best_model_xgb = pickle.load(file)\n","\n","print(\"\\nTesting Performances...Please wait\")\n","best_model_lr.fit(X_train_oversampled, y_train_oversampled)\n","predicted_probs = best_model_lr.predict_proba(X_test_transformed)[:, 1]\n","lr_performance = roc_auc_score(y_test, predicted_probs)\n","\n","best_model_dt.fit(X_train_oversampled, y_train_oversampled)\n","predicted_probs = best_model_dt.predict_proba(X_test_transformed)[:, 1]\n","dt_performance = roc_auc_score(y_test, predicted_probs)\n","\n","best_model_knn.fit(X_train_oversampled, y_train_oversampled)\n","predicted_probs = best_model_knn.predict_proba(X_test_transformed)[:, 1]\n","knn_performance = roc_auc_score(y_test, predicted_probs)\n","\n","best_model_rf.fit(X_train_oversampled, y_train_oversampled)\n","predicted_probs = best_model_rf.predict_proba(X_test_transformed)[:, 1]\n","rf_performance = roc_auc_score(y_test, predicted_probs)\n","\n","best_model_xgb.fit(X_train_oversampled, y_train_oversampled_encoded)\n","predicted_probs = best_model_xgb.predict_proba(X_test_transformed)[:, 1]\n","xgb_performance = roc_auc_score(y_test, predicted_probs)\n","\n","# Test performance of the models are\n","print(f\"Logistic Regression Test ROCAUC: {lr_performance}\")\n","print(f\"Decision Tree Test ROCAUC: {dt_performance}\")\n","print(f\"KNN Test ROCAUC: {knn_performance}\")\n","print(f\"Random Forest Test ROCAUC: {rf_performance}\")\n","print(f\"XGBoost Test ROCAUC: {xgb_performance}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"S3GdDoiOVmQg","executionInfo":{"status":"ok","timestamp":1715576672542,"user_tz":-480,"elapsed":7978,"user":{"displayName":"Eric Lu","userId":"10677817205924610613"}},"outputId":"c237db04-9a50-4814-f581-183c5e04c6e2"},"execution_count":11,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Testing Performances...Please wait\n","Logistic Regression Test ROCAUC: 0.7281135531135531\n","Decision Tree Test ROCAUC: 0.46451465201465203\n","KNN Test ROCAUC: 0.49203296703296706\n","Random Forest Test ROCAUC: 0.8086080586080586\n","XGBoost Test ROCAUC: 0.7736263736263735\n"]}]}]} \ No newline at end of file diff --git a/model_training_top_36_features.py b/model_training_top_36_features.py new file mode 100644 index 0000000..f453680 --- /dev/null +++ b/model_training_top_36_features.py @@ -0,0 +1,275 @@ +import pandas as pd +import numpy as np +import warnings +import seaborn as sns +import matplotlib.pyplot as plt +from imblearn.over_sampling import SMOTENC +from sklearn.model_selection import train_test_split +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, make_scorer +# import sklearn.metrics as skm +from sklearn.neighbors import KNeighborsClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.tree import DecisionTreeClassifier +from sklearn.svm import SVC +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OrdinalEncoder, LabelEncoder +from sklearn.compose import ColumnTransformer +from sklearn.model_selection import cross_val_score, GridSearchCV +import pickle +import optuna +from xgboost import XGBClassifier + +warnings.filterwarnings('ignore') + +# Import the files from data_preprocessing. +X_train_oversampled = pd.read_csv("storage_files/X_train_oversampled.csv") +X_validate_transformed = pd.read_csv("storage_files/X_validate_transformed.csv") +X_test_transformed = pd.read_csv("storage_files/X_test_transformed.csv") +y_train_oversampled = pd.read_csv("storage_files/y_train_oversampled.csv") +y_validate = pd.read_csv("storage_files/y_validate.csv") +y_test = pd.read_csv("storage_files/y_test.csv") +importance_table = pd.read_csv("storage_files/importance_table.csv") + +f = open('storage_files/max_no_features.txt','r') +max_no_features = int(f.readline()) +f.close() +max_features = list(importance_table[:max_no_features]['Feature']) +X_train_oversampled = X_train_oversampled[max_features] +X_validate_transformed = X_validate_transformed[max_features] +X_test_transformed = X_test_transformed[max_features] + +# Logistic Regression +print('\nLogistic Regression started.') +def objective_function(trial): + C = trial.suggest_float('C', 0.1, 10, log=True) + penalty = trial.suggest_categorical('penalty', ['l1', 'l2']) + + model = LogisticRegression( + C=C, + penalty=penalty, + solver='liblinear', + n_jobs=-1 + ) + metrics = ['roc_auc','accuracy','precison','recall','f1-score'] + # Using cross_val_score to get the average precision score for each fold + scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc') + roc_auc = np.mean(scores) + # Printing intermediate results + print(f"Trial {trial.number}, C: {C}, penalty: {penalty}, ROC-AUC: {roc_auc}") + return roc_auc + + +study_lr = optuna.create_study(direction="maximize") +study_lr.optimize(objective_function, n_trials=100) + +best_params_lr = study_lr.best_params +print("Best Parameters: ", best_params_lr) +print("Best ROC-AUC Score: ", study_lr.best_value) + +# Create and save model +best_model_lr = LogisticRegression(**best_params_lr, solver='liblinear', n_jobs=-1) +with open('storage_files/best_models_lr_36_features.pkl', 'wb') as file: + pickle.dump(best_model_lr, file) + +# Decision Tree +print('\nDecision Tree started.') +def objective_function(trial): + max_depth = trial.suggest_int('max_depth', 1, 10) + min_samples_split = trial.suggest_int('min_samples_split', 2, 10) + min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10) + criterion = trial.suggest_categorical('criterion', ['gini', 'entropy']) + + model = DecisionTreeClassifier( + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + criterion=criterion + ) + metrics = ['roc_auc','accuracy','precison','recall','f1-score'] + # Using cross_val_score to get the average precision score for each fold + scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc') + roc_auc = np.mean(scores) + # Printing intermediate results + print(f"Trial {trial.number}, max_depth: {max_depth}, min_samples_split: {min_samples_split}, " + f"min_samples_leaf: {min_samples_leaf}, criterion: {criterion}, ROC-AUC: {roc_auc}") + return roc_auc + +study_dt = optuna.create_study(direction="maximize") +study_dt.optimize(objective_function, n_trials=100) + +best_params_dt = study_dt.best_params +print("Best Parameters: ", best_params_dt) +print("Best ROC-AUC Score: ", study_dt.best_value) + +# Create and save model +best_model_dt = DecisionTreeClassifier(**best_params_dt) +with open('storage_files/best_models_dt_36_features.pkl', 'wb') as file: + pickle.dump(best_model_dt, file) + +# KNN +print('\nKNN started') +def objective_function(trial): + n_neighbors = trial.suggest_int('n_neighbors', 1, 10) + weights = trial.suggest_categorical('weights', ['uniform', 'distance']) + p = trial.suggest_int('p', 1, 5) + metric = trial.suggest_categorical('metric', ['euclidean', 'manhattan', 'minkowski']) + + model = KNeighborsClassifier( + n_neighbors=n_neighbors, + weights=weights, + p=p, + metric=metric + ) + metrics = ['roc_auc','accuracy','precison','recall','f1-score'] + # Using cross_val_score to get the average precision score for each fold + scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc') + roc_auc = np.mean(scores) + # Printing intermediate results + print(f"Trial {trial.number}, n_neighbors: {n_neighbors}, weights: {weights}, p: {p}, metric: {metric}, " + f"ROC-AUC: {roc_auc}") + return roc_auc + + +study_knn = optuna.create_study(direction="maximize") +study_knn.optimize(objective_function, n_trials=100) + +best_params_knn = study_knn.best_params +print("Best Parameters: ", best_params_knn) +print("Best ROC-AUC Score: ", study_knn.best_value) + +# Create and save model +best_model_knn = KNeighborsClassifier(**best_params_knn) +with open('storage_files/best_models_knn_36_features.pkl', 'wb') as file: + pickle.dump(best_model_knn, file) + +# Random Forest +print('\nRandom Forest started') +def objective_function(trial): + n_estimators = trial.suggest_int('n_estimators', 2, 50) + max_depth = trial.suggest_int('max_depth', 1, 10) + min_samples_split = trial.suggest_int('min_samples_split', 2, 10) + min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10) + + model = RandomForestClassifier( + n_estimators=n_estimators, + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + n_jobs=-1 + ) + + # Using cross_val_score to get the average ROC-AUC score for each fold + scores = cross_val_score(model, X_train_oversampled, y_train_oversampled, cv=5, scoring='roc_auc') + roc_auc = np.mean(scores) + # Printing intermediate results + print(f"Trial {trial.number}, n_estimators: {n_estimators}, max_depth: {max_depth}, " + f"min_samples_split: {min_samples_split}, min_samples_leaf: {min_samples_leaf}, ROC-AUC: {roc_auc}") + return roc_auc + + +study_rf = optuna.create_study(direction="maximize") +study_rf.optimize(objective_function, n_trials=100) + +best_params_rf = study_rf.best_params +print("Best Parameters: ", best_params_rf) +print("Best ROC-AUC: Score: ", study_rf.best_value) + +# Create and save model +best_model_rf = RandomForestClassifier(**best_params_rf, n_jobs=-1) +with open('storage_files/best_models_rf_36_features.pkl', 'wb') as file: + pickle.dump(best_model_rf, file) + +label_encoder = LabelEncoder() +# Fit the encoder and transform the target variable +y_train_oversampled_encoded = label_encoder.fit_transform(y_train_oversampled) +# This suppresses printing logs +optuna.logging.set_verbosity(optuna.logging.WARNING) + +# XGBoost +print('\nXGBoost started') +def objective_function(trial): + n_estimators = trial.suggest_int('n_estimators', 2, 50) + max_depth = trial.suggest_int('max_depth', 1, 10) + learning_rate = trial.suggest_float('learning_rate', 0.001, 0.9, log=True) + min_child_weight = trial.suggest_int('min_child_weight', 1, 10) + subsample = trial.suggest_float('subsample', 0.5, 1.0) + colsample_bytree = trial.suggest_float('colsample_bytree', 0.5, 1.0) + gamma = trial.suggest_float('gamma', 0, 1.0) + reg_alpha = trial.suggest_float('reg_alpha', 0, 1) + reg_lambda = trial.suggest_float('reg_lambda', 0, 1) + + model = XGBClassifier( + n_estimators=n_estimators, + max_depth=max_depth, + learning_rate=learning_rate, + min_child_weight=min_child_weight, + subsample=subsample, + colsample_bytree=colsample_bytree, + gamma=gamma, + reg_alpha=reg_alpha, + reg_lambda=reg_lambda, + use_label_encoder=False, + n_jobs=-1 + ) + + # Using cross_val_score to get the average ROC-AUC score for each fold + scores = cross_val_score(model, X_train_oversampled, y_train_oversampled_encoded, cv=5, scoring='roc_auc') + roc_auc = np.mean(scores) + # Printing intermediate results + print(f"Trial {trial.number}, n_estimators: {n_estimators}, max_depth: {max_depth}, learning_rate: {learning_rate}," + f"min_child_weight: {min_child_weight}, subsample: {subsample}, colsample_bytree: {colsample_bytree}, " + f"gamma: {gamma}, reg_alpha: {reg_alpha}, reg_lambda: {reg_lambda}, ROC-AUC: {roc_auc}") + return roc_auc + + +study_xgb = optuna.create_study(direction="maximize") +study_xgb.optimize(objective_function, n_trials=100) + +best_params_xgb = study_xgb.best_params +print("Best Parameters: ", best_params_xgb) +print("Best ROC-AUC Score: ", study_xgb.best_value) + +best_model_xgb = XGBClassifier(**best_params_xgb, use_label_encoder=False, n_jobs=-1) +with open('storage_files/best_models_xgb_36_features.pkl', 'wb') as file: + pickle.dump(best_model_xgb, file) + +# Model selection - Compare Performance +with open('storage_files/best_models_lr_36_features.pkl', 'rb') as file: + best_model_lr = pickle.load(file) +with open('storage_files/best_models_dt_36_features.pkl', 'rb') as file: + best_model_dt = pickle.load(file) +with open('storage_files/best_models_knn_36_features.pkl', 'rb') as file: + best_model_knn = pickle.load(file) +with open('storage_files/best_models_rf_36_features.pkl', 'rb') as file: + best_model_rf = pickle.load(file) +with open('storage_files/best_models_xgb_36_features.pkl', 'rb') as file: + best_model_xgb = pickle.load(file) + +print("\nTesting Performances...Please wait") +best_model_lr.fit(X_train_oversampled, y_train_oversampled) +predicted_probs = best_model_lr.predict_proba(X_test_transformed)[:, 1] +lr_performance = roc_auc_score(y_test, predicted_probs) + +best_model_dt.fit(X_train_oversampled, y_train_oversampled) +predicted_probs = best_model_dt.predict_proba(X_test_transformed)[:, 1] +dt_performance = roc_auc_score(y_test, predicted_probs) + +best_model_knn.fit(X_train_oversampled, y_train_oversampled) +predicted_probs = best_model_knn.predict_proba(X_test_transformed)[:, 1] +knn_performance = roc_auc_score(y_test, predicted_probs) + +best_model_rf.fit(X_train_oversampled, y_train_oversampled) +predicted_probs = best_model_rf.predict_proba(X_test_transformed)[:, 1] +rf_performance = roc_auc_score(y_test, predicted_probs) + +best_model_xgb.fit(X_train_oversampled, y_train_oversampled_encoded) +predicted_probs = best_model_xgb.predict_proba(X_test_transformed)[:, 1] +xgb_performance = roc_auc_score(y_test, predicted_probs) + +# Test performance of the models are +print(f"Logistic Regression Test ROCAUC: {lr_performance}") +print(f"Decision Tree Test ROCAUC: {dt_performance}") +print(f"KNN Test ROCAUC: {knn_performance}") +print(f"Random Forest Test ROCAUC: {rf_performance}") +print(f"XGBoost Test ROCAUC: {xgb_performance}") diff --git a/model_training_top_8_features.py b/model_training_top_8_features.py index 33f0c28..6aa3d45 100644 --- a/model_training_top_8_features.py +++ b/model_training_top_8_features.py @@ -7,7 +7,7 @@ https://colab.research.google.com/drive/1NYXG63Ag_lYYssS6FqQIyn3S0zUoicBj """ -! pip install optuna +# ! pip install optuna import pandas as pd import numpy as np @@ -285,4 +285,4 @@ def objective_function(trial): print(f"Decision Tree Test ROCAUC: {dt_performance}") print(f"KNN Test ROCAUC: {knn_performance}") print(f"Random Forest Test ROCAUC: {rf_performance}") -print(f"XGBoost Test ROCAUC: {xgb_performance}") \ No newline at end of file +print(f"XGBoost Test ROCAUC: {xgb_performance}") diff --git a/result_df.pkl b/result_df.pkl deleted file mode 100644 index 72a9dc2..0000000 Binary files a/result_df.pkl and /dev/null differ diff --git a/storage_files/best_models_dt.pkl b/storage_files/best_models_dt.pkl new file mode 100644 index 0000000..fc06bfd Binary files /dev/null and b/storage_files/best_models_dt.pkl differ diff --git a/storage_files/best_models_dt_36_features.pkl b/storage_files/best_models_dt_36_features.pkl new file mode 100644 index 0000000..cf220e2 Binary files /dev/null and b/storage_files/best_models_dt_36_features.pkl differ diff --git a/storage_files/best_models_knn.pkl b/storage_files/best_models_knn.pkl new file mode 100644 index 0000000..250bab2 Binary files /dev/null and b/storage_files/best_models_knn.pkl differ diff --git a/storage_files/best_models_knn_36_features.pkl b/storage_files/best_models_knn_36_features.pkl new file mode 100644 index 0000000..4e80406 Binary files /dev/null and b/storage_files/best_models_knn_36_features.pkl differ diff --git a/best_models_lr.pkl b/storage_files/best_models_lr.pkl similarity index 80% rename from best_models_lr.pkl rename to storage_files/best_models_lr.pkl index 0c31b54..bdc2d3f 100644 Binary files a/best_models_lr.pkl and b/storage_files/best_models_lr.pkl differ diff --git a/storage_files/best_models_lr_36_features.pkl b/storage_files/best_models_lr_36_features.pkl new file mode 100644 index 0000000..590fd31 Binary files /dev/null and b/storage_files/best_models_lr_36_features.pkl differ diff --git a/storage_files/best_models_rf.pkl b/storage_files/best_models_rf.pkl new file mode 100644 index 0000000..68b9b61 Binary files /dev/null and b/storage_files/best_models_rf.pkl differ diff --git a/storage_files/best_models_rf_36_features.pkl b/storage_files/best_models_rf_36_features.pkl new file mode 100644 index 0000000..5412039 Binary files /dev/null and b/storage_files/best_models_rf_36_features.pkl differ diff --git a/storage_files/best_models_xgb.pkl b/storage_files/best_models_xgb.pkl new file mode 100644 index 0000000..bab787f Binary files /dev/null and b/storage_files/best_models_xgb.pkl differ diff --git a/storage_files/best_models_xgb_36_features.pkl b/storage_files/best_models_xgb_36_features.pkl new file mode 100644 index 0000000..1510fef Binary files /dev/null and b/storage_files/best_models_xgb_36_features.pkl differ diff --git a/storage_files/max_no_features.txt b/storage_files/max_no_features.txt new file mode 100644 index 0000000..7facc89 --- /dev/null +++ b/storage_files/max_no_features.txt @@ -0,0 +1 @@ +36 diff --git a/storage_files/result_df.csv b/storage_files/result_df.csv index eade839..1238050 100644 --- a/storage_files/result_df.csv +++ b/storage_files/result_df.csv @@ -1,93 +1,21 @@ -Features_Removed,no_features_used,ROC_Score -None,92,0.7359890109890109 -pretaxProfitMargin_percentage_change,91,0.7729395604395605 -debtEquityRatio_percentage_change,90,0.7572344322344322 -effectiveTaxRate,89,0.7139652014652016 -operatingProfitMargin_percentage_change,88,0.7190018315018315 -cashFlowToDebtRatio_percentage_change,87,0.7364468864468865 -priceToOperatingCashFlowsRatio_percentage_change,86,0.7312271062271063 -netProfitMargin_percentage_change,85,0.7313644688644688 -receivablesTurnover_percentage_change,84,0.7323717948717948 -capitalExpenditureCoverageRatio_percentage_change,83,0.7452380952380953 -returnOnEquity_percentage_change,82,0.7571886446886447 -totalDebtToCapitalization,81,0.7570512820512821 -currentRatio_percentage_change,80,0.7378205128205129 -assetTurnover_percentage_change,79,0.7670787545787545 -daysOfInventoryOutstanding_percentage_change,78,0.7471611721611722 -pretaxProfitMargin,77,0.7534798534798535 -returnOnCapitalEmployed,76,0.7673534798534799 -debtRatio_percentage_change,75,0.7697344322344323 -fixedAssetTurnover_percentage_change,74,0.7413919413919413 -cashPerShare,73,0.7549450549450549 -currentRatio,72,0.7477106227106226 -priceToSalesRatio_percentage_change,71,0.7076007326007326 -companyEquityMultiplier,70,0.7322344322344323 -companyEquityMultiplier_percentage_change,69,0.7593406593406593 -enterpriseValueMultiple,68,0.7249084249084249 -operatingCashFlowPerShare_percentage_change,67,0.7617216117216117 -returnOnAssets_percentage_change,66,0.7673534798534798 -priceEarningsRatio_percentage_change,65,0.771978021978022 -interestCoverage,64,0.7214285714285714 -netProfitMargin,63,0.7382326007326008 -freeCashFlowPerShare_percentage_change,62,0.7515567765567767 -priceBookValueRatio,61,0.7466575091575092 -cashFlowToDebtRatio,60,0.7618589743589743 -priceEarningsToGrowthRatio,59,0.7732142857142856 -priceEarningsRatio,58,0.7529761904761906 -freeCashFlowOperatingCashFlowRatio,57,0.7351648351648352 -debtEquityRatio,56,0.756364468864469 -inventoryTurnover_percentage_change,55,0.7570970695970697 -longTermDebtToCapitalization_percentage_change,54,0.7477106227106227 -returnOnEquity,53,0.777014652014652 -priceToOperatingCashFlowsRatio,52,0.7682692307692308 -payoutRatio_percentage_change,51,0.7653388278388279 -longTermDebtToCapitalization,50,0.7443223443223443 -daysOfSalesOutstanding_percentage_change,49,0.7629578754578754 -shortTermCoverageRatios,48,0.7696886446886447 -payoutRatio,47,0.7533424908424908 -receivablesTurnover,46,0.81753663003663 -ebtPerEbit,45,0.8005952380952381 -payablesTurnover_percentage_change,44,0.8079212454212454 -effectiveTaxRate_percentage_change,43,0.8159798534798535 -freeCashFlowOperatingCashFlowRatio_percentage_change,42,0.7576923076923077 -netIncomePerEBT,41,0.7802197802197802 -quickRatio,40,0.7991300366300366 -operatingProfitMargin,39,0.8007326007326007 -inventoryTurnover,38,0.7758241758241757 -dividendPaidAndCapexCoverageRatio_percentage_change,37,0.7972527472527472 -cashPerShare_percentage_change,36,0.8312728937728937 -daysOfSalesOutstanding,35,0.7567307692307692 -shortTermCoverageRatios_percentage_change,34,0.7941849816849818 -grossProfitMargin,33,0.7564102564102565 -returnOnAssets,32,0.8097527472527473 -dividendPaidAndCapexCoverageRatio,31,0.779532967032967 -payablesTurnover,30,0.759065934065934 -fixedAssetTurnover,29,0.7906593406593406 -daysOfInventoryOutstanding,28,0.7599358974358974 -grossProfitMargin_percentage_change,27,0.7566391941391942 -priceEarningsToGrowthRatio_percentage_change,26,0.7619047619047619 -priceToFreeCashFlowsRatio,25,0.7828754578754579 -interestCoverage_percentage_change,24,0.7837454212454211 -netIncomePerEBT_percentage_change,23,0.7861263736263737 -daysOfPayablesOutstanding,22,0.8003663003663004 -operatingCashFlowPerShare,21,0.7879120879120879 -enterpriseValueMultiple_percentage_change,20,0.8060439560439561 -priceToSalesRatio,19,0.7920787545787545 -industry,18,0.7994505494505494 -dividendYield_percentage_change,17,0.8179029304029304 -operatingCashFlowSalesRatio,16,0.8171703296703298 -operatingCycle_percentage_change,15,0.7935897435897437 -priceToFreeCashFlowsRatio_percentage_change,14,0.8262820512820513 -ebtPerEbit_percentage_change,13,0.8076465201465202 -capitalExpenditureCoverageRatio,12,0.7407967032967032 -symbol,11,0.7027930402930403 -assetTurnover,10,0.7176739926739927 -sector,9,0.7187271062271061 -debtRatio,8,0.6842948717948718 -adjDividend,7,0.6129578754578755 -freeCashFlowPerShare,6,0.6671703296703297 -interestRate_percentage_change,5,0.6421245421245421 -dividendYield,4,0.6248626373626374 -year,3,0.6013736263736262 -interestRate,2,0.5380036630036631 -dps_growth,1,0.6287545787545789 +Features_Removed,no_features_used,ROC_Score,f1,accuracy,precision,recall +None,92,0.7497252747252747,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,91,0.7331043956043957,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,90,0.7298534798534798,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,89,0.7482142857142857,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,88,0.7554945054945056,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,87,0.7548534798534798,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,86,0.7301282051282051,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,85,0.7346611721611722,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,84,0.7592948717948719,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,83,0.7499542124542126,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,82,0.716941391941392,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,81,0.7309523809523809,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,80,0.7684981684981684,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,79,0.7733058608058608,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,78,0.7270604395604395,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,77,0.7263736263736263,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,76,0.763598901098901,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,75,0.7308150183150184,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,74,0.7532051282051282,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847 +None,73,0.759981684981685,0.9739130434782609,0.9498956158663883,0.9634408602150538,0.9846153846153847