catboost_randomizedsearch_stacking_smartfactory_yumin.py

# -*- coding: utf-8 -*-
"""catBoost.RandomizedSearch-stacking-smartFactory-Yumin.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/19eCa0FTXfevKxWolt72GaGa5GqGgOXkF
"""

!pip install catboost 
!pip install randomsearch
!pip install optuna

import pandas as pd
import random
import os
import numpy as np
import matplotlib.pyplot as plt   
import seaborn as sns 

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

import tensorflow as tf
from tensorflow.keras import regularizers
from keras.models import Sequential
from keras.layers import Dense,Dropout, Activation

from sklearn.metrics import f1_score, classification_report
from catboost import CatBoostClassifier


import optuna
from optuna.samplers import TPESampler
from optuna import Trial

from google.colab import drive  # GoogleDrive mount 
drive.mount('/content/drive')

"""# 1.preprocessing
1. Label Encoding: categorical values `LINE`, `PRODUCT_CODE`
2. Missing values: fillna(0) 
3. scaling: StandardScaler
"""

train_df = pd.read_csv('/content/drive/MyDrive/train.csv')
test_df = pd.read_csv('/content/drive/MyDrive/test.csv')
submit = pd.read_csv('/content/drive/MyDrive/sample_submission.csv')

train_x = train_df.drop(columns=['PRODUCT_ID', 'TIMESTAMP', 'Y_Class','Y_Quality'])
train_y = train_df['Y_Class']
test_x = test_df.drop(columns=['PRODUCT_ID', 'TIMESTAMP'])   

# 1) qualitative to quantitative
qual_col = ['LINE', 'PRODUCT_CODE']
for i in qual_col:
    le = LabelEncoder()    # TRY one-hot encoding 
    le = le.fit(train_x[i])
    train_x[i] = le.transform(train_x[i])
    for label in np.unique(test_x[i]): 
        if label not in le.classes_: 
            le.classes_ = np.append(le.classes_, label)
    test_x[i] = le.transform(test_x[i]) 

# 2) Missing Values 
train_x = train_x.fillna(0)
test_x = test_x.fillna(0)


# 3) scaling: only `X_???` values (continuous)
from sklearn.preprocessing import StandardScaler
Xs = train_x.select_dtypes(include=float).iloc[:,1:].columns.tolist()
scaler = StandardScaler().fit(train_x.loc[:, Xs])
train_x.loc[:, Xs] = scaler.transform(train_x.loc[:, Xs])
test_x.loc[:, Xs] = scaler.transform(test_x.loc[:, Xs])

X_train, X_test, y_train, y_test=train_test_split(train_x,train_y,test_size=0.3,random_state=37)

from sklearn.model_selection import RandomizedSearchCV

# Setting parameter
param_distributions = {
    'learning_rate': [0.001, 0.01, 0.1],
    'depth': [4, 6, 8],
    'iterations': [500, 1000],
    'l2_leaf_reg': [0, 1, 5],
    'bagging_temperature': [0, 1, 5],
}

# Model define
catboost_model = CatBoostClassifier(
    verbose=False, 
    random_state=37
)

# perform random search
random_search = RandomizedSearchCV(
    catboost_model, 
    param_distributions=param_distributions,
    cv=5,
    n_iter=20,
    random_state=37,
    n_jobs=4, 
    scoring='accuracy'
)

# model fitting
random_search.fit(X_train, y_train)

# print best hyperparameters
print("Best hyperparameters: ", random_search.best_params_)

# use best hyperparameters to train final model
catboost_model = CatBoostClassifier(
            verbose=False,
            random_state=37,
            **random_search.best_params_
)

catboost_model.fit(X_train, y_train)

preds = catboost_model.predict(X_test)



print(classification_report(preds, y_test))

from sklearn.metrics import classification_report
print(classification_report(preds, y_test))

preds.reshape([1,310])

"""# 4.Ensemble
- 35~42
"""

from catboost import CatBoostClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import StackingClassifier
from sklearn.linear_model import LogisticRegression

models = [
   
    ('gb', GradientBoostingClassifier(random_state=37)),
    ('catB', CatBoostClassifier( 
    **param_distributions, verbose=False, random_state=37
)),
   
]

stack = StackingClassifier(
    estimators=models,
    final_estimator=LogisticRegression(),
    cv=5,
    n_jobs=-1,
    passthrough=True
)

# use X_test as validation set
stack.fit(train_x, train_y)
train_stack = np.concatenate(
    [model.predict_proba(train_x) for model in models], axis=1)
val_stack = np.concatenate(
    [model.predict_proba(X_test) for model in models], axis=1)

# meta model training
meta_model = LogisticRegression().fit(train_stack, train_y)
preds = meta_model.predict(val_stack)

preds
sns.countplot(x=preds);

submit = pd.read_csv('/content/drive/MyDrive/sample_submission.csv')
submit['Y_Class'] = preds

submit.to_csv('rsCV_catboost_submission.csv', index=False)



# #   Params:  {'iterations': 848, 'learning_rate': 0.030969090455030726, 'depth': 8, 'l2_leaf_reg': 6.761650264581644e-08, 'bootstrap_type': 'Bayesian', 'random_strength': 5.097812627692342e-07, 'bagging_temperature': 7.206905770161392, 'od_type': 'IncToDec', 'od_wait': 49}

# model = CatBoostClassifier( 
#     iterations= 960, 
#                                        learning_rate= 0.0022686226693627966,
#                                        depth= 4, 
#                                        l2_leaf_reg= 0.00025235896782569236, 
#                                        bootstrap_type= 'Bayesian', 
#                                        random_strength= 0.07344055855215284, 
#                                        bagging_temperature= 0.5154099595769013, 
#                                        od_type= 'IncToDec', 
#                                        od_wait=34)

# b = pd.read_csv('catB_noSMOTE-smartFactorySubmission.csv')
# sns.countplot(x = b['Y_Class']);