classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://deep-deep-deep.tistory.com/160"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings \n",
    "warnings.filterwarnings('ignore')\n",
    "pd.options.display.float_format = '{:.5f}'.format\n",
    "\n",
    "# 한글 폰트 문제\n",
    "from matplotlib import font_manager, rc\n",
    "font_path = \"C:/Windows/Fonts/malgun.ttf\"\n",
    "font = font_manager.FontProperties(fname=font_path).get_name()\n",
    "rc('font', family=font)\n",
    "\n",
    "# 모델 관련\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from xgboost import plot_importance\n",
    "from catboost import CatBoostClassifier\n",
    "\n",
    "# 인코딩\n",
    "from category_encoders import BinaryEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>avg_score</th>\n",
       "      <th>num_usecard</th>\n",
       "      <th>monthly_card_spend</th>\n",
       "      <th>monthly_bk_loan</th>\n",
       "      <th>monthly_cd_loan</th>\n",
       "      <th>monthly_installments_loan</th>\n",
       "      <th>monthly_insurance_loan</th>\n",
       "      <th>monthly_sbk_loan</th>\n",
       "      <th>loan_commitment</th>\n",
       "      <th>inst_rep_loanb</th>\n",
       "      <th>...</th>\n",
       "      <th>credit_card_installments_payment</th>\n",
       "      <th>monthly_spend</th>\n",
       "      <th>cluster</th>\n",
       "      <th>city</th>\n",
       "      <th>ages</th>\n",
       "      <th>sex</th>\n",
       "      <th>avg_rat</th>\n",
       "      <th>monthly_loan</th>\n",
       "      <th>credit_card_payment</th>\n",
       "      <th>debit_card_payment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>740</td>\n",
       "      <td>1.99000</td>\n",
       "      <td>810000</td>\n",
       "      <td>18500000</td>\n",
       "      <td>620000</td>\n",
       "      <td>2110000</td>\n",
       "      <td>680000</td>\n",
       "      <td>1320000</td>\n",
       "      <td>3800000</td>\n",
       "      <td>10520000</td>\n",
       "      <td>...</td>\n",
       "      <td>83000</td>\n",
       "      <td>727000</td>\n",
       "      <td>2</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>24940000</td>\n",
       "      <td>254000</td>\n",
       "      <td>473000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>782</td>\n",
       "      <td>1.75000</td>\n",
       "      <td>790000</td>\n",
       "      <td>24150000</td>\n",
       "      <td>680000</td>\n",
       "      <td>2300000</td>\n",
       "      <td>790000</td>\n",
       "      <td>1160000</td>\n",
       "      <td>5420000</td>\n",
       "      <td>12220000</td>\n",
       "      <td>...</td>\n",
       "      <td>83000</td>\n",
       "      <td>707000</td>\n",
       "      <td>2</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>30880000</td>\n",
       "      <td>278000</td>\n",
       "      <td>429000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>782</td>\n",
       "      <td>1.74000</td>\n",
       "      <td>790000</td>\n",
       "      <td>24240000</td>\n",
       "      <td>680000</td>\n",
       "      <td>2300000</td>\n",
       "      <td>790000</td>\n",
       "      <td>1210000</td>\n",
       "      <td>5350000</td>\n",
       "      <td>12280000</td>\n",
       "      <td>...</td>\n",
       "      <td>78000</td>\n",
       "      <td>712000</td>\n",
       "      <td>2</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>31000000</td>\n",
       "      <td>267000</td>\n",
       "      <td>445000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>782</td>\n",
       "      <td>1.75000</td>\n",
       "      <td>860000</td>\n",
       "      <td>24570000</td>\n",
       "      <td>680000</td>\n",
       "      <td>2300000</td>\n",
       "      <td>790000</td>\n",
       "      <td>1230000</td>\n",
       "      <td>5380000</td>\n",
       "      <td>12320000</td>\n",
       "      <td>...</td>\n",
       "      <td>89000</td>\n",
       "      <td>771000</td>\n",
       "      <td>2</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>31390000</td>\n",
       "      <td>292000</td>\n",
       "      <td>479000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>781</td>\n",
       "      <td>1.75000</td>\n",
       "      <td>820000</td>\n",
       "      <td>24830000</td>\n",
       "      <td>700000</td>\n",
       "      <td>2320000</td>\n",
       "      <td>770000</td>\n",
       "      <td>1250000</td>\n",
       "      <td>5430000</td>\n",
       "      <td>12420000</td>\n",
       "      <td>...</td>\n",
       "      <td>82000</td>\n",
       "      <td>738000</td>\n",
       "      <td>2</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>31730000</td>\n",
       "      <td>285000</td>\n",
       "      <td>453000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3019</th>\n",
       "      <td>818</td>\n",
       "      <td>0.18000</td>\n",
       "      <td>200000</td>\n",
       "      <td>58460000</td>\n",
       "      <td>230000</td>\n",
       "      <td>260000</td>\n",
       "      <td>1610000</td>\n",
       "      <td>290000</td>\n",
       "      <td>8600000</td>\n",
       "      <td>18710000</td>\n",
       "      <td>...</td>\n",
       "      <td>43000</td>\n",
       "      <td>157000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80대</td>\n",
       "      <td>여자</td>\n",
       "      <td>3</td>\n",
       "      <td>69610000</td>\n",
       "      <td>82000</td>\n",
       "      <td>75000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3020</th>\n",
       "      <td>818</td>\n",
       "      <td>0.18000</td>\n",
       "      <td>190000</td>\n",
       "      <td>58330000</td>\n",
       "      <td>230000</td>\n",
       "      <td>270000</td>\n",
       "      <td>1610000</td>\n",
       "      <td>310000</td>\n",
       "      <td>8550000</td>\n",
       "      <td>18750000</td>\n",
       "      <td>...</td>\n",
       "      <td>39000</td>\n",
       "      <td>151000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80대</td>\n",
       "      <td>여자</td>\n",
       "      <td>3</td>\n",
       "      <td>69500000</td>\n",
       "      <td>82000</td>\n",
       "      <td>69000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3021</th>\n",
       "      <td>818</td>\n",
       "      <td>0.18000</td>\n",
       "      <td>210000</td>\n",
       "      <td>58190000</td>\n",
       "      <td>240000</td>\n",
       "      <td>290000</td>\n",
       "      <td>1610000</td>\n",
       "      <td>430000</td>\n",
       "      <td>8630000</td>\n",
       "      <td>18760000</td>\n",
       "      <td>...</td>\n",
       "      <td>49000</td>\n",
       "      <td>161000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80대</td>\n",
       "      <td>여자</td>\n",
       "      <td>3</td>\n",
       "      <td>69510000</td>\n",
       "      <td>87000</td>\n",
       "      <td>74000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3022</th>\n",
       "      <td>819</td>\n",
       "      <td>0.18000</td>\n",
       "      <td>200000</td>\n",
       "      <td>58420000</td>\n",
       "      <td>240000</td>\n",
       "      <td>300000</td>\n",
       "      <td>1610000</td>\n",
       "      <td>470000</td>\n",
       "      <td>8570000</td>\n",
       "      <td>18690000</td>\n",
       "      <td>...</td>\n",
       "      <td>44000</td>\n",
       "      <td>156000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80대</td>\n",
       "      <td>여자</td>\n",
       "      <td>3</td>\n",
       "      <td>69750000</td>\n",
       "      <td>80000</td>\n",
       "      <td>76000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3023</th>\n",
       "      <td>819</td>\n",
       "      <td>0.18000</td>\n",
       "      <td>200000</td>\n",
       "      <td>58170000</td>\n",
       "      <td>250000</td>\n",
       "      <td>300000</td>\n",
       "      <td>1640000</td>\n",
       "      <td>430000</td>\n",
       "      <td>8590000</td>\n",
       "      <td>18670000</td>\n",
       "      <td>...</td>\n",
       "      <td>44000</td>\n",
       "      <td>156000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80대</td>\n",
       "      <td>여자</td>\n",
       "      <td>3</td>\n",
       "      <td>69510000</td>\n",
       "      <td>81000</td>\n",
       "      <td>75000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3024 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      avg_score  num_usecard  monthly_card_spend  monthly_bk_loan  \\\n",
       "0           740      1.99000              810000         18500000   \n",
       "1           782      1.75000              790000         24150000   \n",
       "2           782      1.74000              790000         24240000   \n",
       "3           782      1.75000              860000         24570000   \n",
       "4           781      1.75000              820000         24830000   \n",
       "...         ...          ...                 ...              ...   \n",
       "3019        818      0.18000              200000         58460000   \n",
       "3020        818      0.18000              190000         58330000   \n",
       "3021        818      0.18000              210000         58190000   \n",
       "3022        819      0.18000              200000         58420000   \n",
       "3023        819      0.18000              200000         58170000   \n",
       "\n",
       "      monthly_cd_loan  monthly_installments_loan  monthly_insurance_loan  \\\n",
       "0              620000                    2110000                  680000   \n",
       "1              680000                    2300000                  790000   \n",
       "2              680000                    2300000                  790000   \n",
       "3              680000                    2300000                  790000   \n",
       "4              700000                    2320000                  770000   \n",
       "...               ...                        ...                     ...   \n",
       "3019           230000                     260000                 1610000   \n",
       "3020           230000                     270000                 1610000   \n",
       "3021           240000                     290000                 1610000   \n",
       "3022           240000                     300000                 1610000   \n",
       "3023           250000                     300000                 1640000   \n",
       "\n",
       "      monthly_sbk_loan  loan_commitment  inst_rep_loanb  ...  \\\n",
       "0              1320000          3800000        10520000  ...   \n",
       "1              1160000          5420000        12220000  ...   \n",
       "2              1210000          5350000        12280000  ...   \n",
       "3              1230000          5380000        12320000  ...   \n",
       "4              1250000          5430000        12420000  ...   \n",
       "...                ...              ...             ...  ...   \n",
       "3019            290000          8600000        18710000  ...   \n",
       "3020            310000          8550000        18750000  ...   \n",
       "3021            430000          8630000        18760000  ...   \n",
       "3022            470000          8570000        18690000  ...   \n",
       "3023            430000          8590000        18670000  ...   \n",
       "\n",
       "      credit_card_installments_payment  monthly_spend  cluster  city  ages  \\\n",
       "0                                83000         727000        2    서울   20대   \n",
       "1                                83000         707000        2    서울   20대   \n",
       "2                                78000         712000        2    서울   20대   \n",
       "3                                89000         771000        2    서울   20대   \n",
       "4                                82000         738000        2    서울   20대   \n",
       "...                                ...            ...      ...   ...   ...   \n",
       "3019                             43000         157000        0   NaN   80대   \n",
       "3020                             39000         151000        0   NaN   80대   \n",
       "3021                             49000         161000        0   NaN   80대   \n",
       "3022                             44000         156000        0   NaN   80대   \n",
       "3023                             44000         156000        0   NaN   80대   \n",
       "\n",
       "      sex avg_rat monthly_loan credit_card_payment  debit_card_payment  \n",
       "0     NaN       4     24940000              254000              473000  \n",
       "1     NaN       4     30880000              278000              429000  \n",
       "2     NaN       4     31000000              267000              445000  \n",
       "3     NaN       4     31390000              292000              479000  \n",
       "4     NaN       4     31730000              285000              453000  \n",
       "...   ...     ...          ...                 ...                 ...  \n",
       "3019   여자       3     69610000               82000               75000  \n",
       "3020   여자       3     69500000               82000               69000  \n",
       "3021   여자       3     69510000               87000               74000  \n",
       "3022   여자       3     69750000               80000               76000  \n",
       "3023   여자       3     69510000               81000               75000  \n",
       "\n",
       "[3024 rows x 23 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./dataset/gmm_clustering.csv', encoding='utf-8')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_encoding = df[['monthly_spend', 'avg_score', 'cluster', 'city', 'ages']]\n",
    "df_encoding['city'] = df_encoding['city'].fillna('없음')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['서울', '인천', '대전', '대구', '광주', '부산', '울산', '경기', '강원', '충북', '충남',\n",
       "       '전북', '전남', '경북', '경남', '제주', '없음'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_encoding.city.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 바이너리 인코딩(Binary Encoding) - 도시, 나이\n",
    "enc_city = BinaryEncoder(cols='city')\n",
    "enc_age = BinaryEncoder(cols='ages')\n",
    "df_encoding = enc_city.fit_transform(df_encoding)\n",
    "df_encoding = enc_age.fit_transform(df_encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_encoding.drop(['city_0', 'ages_0'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>monthly_spend</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>cluster</th>\n",
       "      <th>city_1</th>\n",
       "      <th>city_2</th>\n",
       "      <th>city_3</th>\n",
       "      <th>city_4</th>\n",
       "      <th>city_5</th>\n",
       "      <th>ages_1</th>\n",
       "      <th>ages_2</th>\n",
       "      <th>ages_3</th>\n",
       "      <th>city</th>\n",
       "      <th>ages</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>727000</td>\n",
       "      <td>740</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>707000</td>\n",
       "      <td>782</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>712000</td>\n",
       "      <td>782</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>771000</td>\n",
       "      <td>782</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>738000</td>\n",
       "      <td>781</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   monthly_spend  avg_score  cluster  city_1  city_2  city_3  city_4  city_5  \\\n",
       "0         727000        740        2       0       0       0       0       1   \n",
       "1         707000        782        2       0       0       0       0       1   \n",
       "2         712000        782        2       0       0       0       0       1   \n",
       "3         771000        782        2       0       0       0       0       1   \n",
       "4         738000        781        2       0       0       0       0       1   \n",
       "\n",
       "   ages_1  ages_2  ages_3 city ages  \n",
       "0       0       0       1   서울  20대  \n",
       "1       0       0       1   서울  20대  \n",
       "2       0       0       1   서울  20대  \n",
       "3       0       0       1   서울  20대  \n",
       "4       0       0       1   서울  20대  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# city, age 붙여넣기\n",
    "df_encoding = pd.concat([df_encoding, df[['city', 'ages']]], axis=1)\n",
    "df_encoding['city'] = df_encoding['city'].fillna('없음')\n",
    "df_encoding.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataframe 컬럼 순서 변경\n",
    "df_encoding = df_encoding[['monthly_spend', 'avg_score', 'city_1', 'city_2',\n",
    "       'city_3', 'city_4', 'city_5', 'ages_1', 'ages_2', 'ages_3',\n",
    "       'city', 'ages', 'cluster']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "city_encoding = df_encoding[['city_1', 'city_2', 'city_3', 'city_4', 'city_5', 'city']].drop_duplicates().reset_index(drop=True)\n",
    "ages_encoding = df_encoding[['ages_1', 'ages_2', 'ages_3', 'ages']].drop_duplicates().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "city_encoding.to_csv('./dataset/city_encoding.csv', encoding='utf-8', index=False)\n",
    "ages_encoding.to_csv('./dataset/ages_encoding.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>monthly_spend</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>city_1</th>\n",
       "      <th>city_2</th>\n",
       "      <th>city_3</th>\n",
       "      <th>city_4</th>\n",
       "      <th>city_5</th>\n",
       "      <th>ages_1</th>\n",
       "      <th>ages_2</th>\n",
       "      <th>ages_3</th>\n",
       "      <th>city</th>\n",
       "      <th>ages</th>\n",
       "      <th>cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>727000</td>\n",
       "      <td>740</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>707000</td>\n",
       "      <td>782</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>712000</td>\n",
       "      <td>782</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>771000</td>\n",
       "      <td>782</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>738000</td>\n",
       "      <td>781</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   monthly_spend  avg_score  city_1  city_2  city_3  city_4  city_5  ages_1  \\\n",
       "0         727000        740       0       0       0       0       1       0   \n",
       "1         707000        782       0       0       0       0       1       0   \n",
       "2         712000        782       0       0       0       0       1       0   \n",
       "3         771000        782       0       0       0       0       1       0   \n",
       "4         738000        781       0       0       0       0       1       0   \n",
       "\n",
       "   ages_2  ages_3 city ages  cluster  \n",
       "0       0       1   서울  20대        2  \n",
       "1       0       1   서울  20대        2  \n",
       "2       0       1   서울  20대        2  \n",
       "3       0       1   서울  20대        2  \n",
       "4       0       1   서울  20대        2  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_xgb = df_encoding.copy()\n",
    "df_xgb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2419, 10) (605, 10)\n"
     ]
    }
   ],
   "source": [
    "# 피쳐와 라벨 분류\n",
    "X_features = df_xgb.iloc[:, :-3]\n",
    "y_labels = df_xgb.iloc[:, -1]\n",
    "\n",
    "# train, test 데이터셋 분리\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_features, y_labels, test_size=0.2, random_state=121)\n",
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBClassifier()\n",
    "param_grid_xgb={'booster' :['gbtree'],\n",
    "                 'silent':[1],\n",
    "                 'max_depth':[5,6],\n",
    "                 'min_child_weight':[1,3,5],\n",
    "                 'gamma':[0,1],\n",
    "                 'nthread':[4],\n",
    "                 'colsample_bytree':[0.5,0.8],\n",
    "                 'colsample_bylevel':[0.9],\n",
    "                 'n_estimators' : [300, 500],\n",
    "                 'objective':['multi:softmax'],\n",
    "                 'learning_rate' : [0.05, 0.1],\n",
    "                 'random_state':[121],\n",
    "                 'subsample' : [0.7, 1.0],\n",
    "                 'eval_metric':['mlogloss', 'merror']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:14:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/learner.cc:627: \n",
      "Parameters: { \"silent\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "kf = StratifiedKFold(n_splits=5, random_state=121, shuffle=True)\n",
    "\n",
    "gcv_xgb = GridSearchCV(xgb, param_grid=param_grid_xgb, cv = kf, scoring='f1_macro', n_jobs=4, refit = True, return_train_score = True)\n",
    "gcv_xgb.fit(X_train, y_train)\n",
    "\n",
    "# 각 파라미터값들에 대한 모델 결과값들이 cv_results_ 객체에 할당됨\n",
    "scores_xgb_df = pd.DataFrame(gcv_xgb.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "최적의 파라미터 : {'booster': 'gbtree', 'colsample_bylevel': 0.9, 'colsample_bytree': 0.5, 'eval_metric': 'mlogloss', 'gamma': 0, 'learning_rate': 0.1, 'max_depth': 5, 'min_child_weight': 1, 'n_estimators': 500, 'nthread': 4, 'objective': 'multi:softmax', 'random_state': 121, 'silent': 1, 'subsample': 1.0}\n",
      "최적의 파라미터로 모델의 정확도 : 0.9916460343338345\n"
     ]
    }
   ],
   "source": [
    "# 최적의 파라미터는 best_params_에 할당되어있음\n",
    "print(f\"최적의 파라미터 : {gcv_xgb.best_params_}\")\n",
    "print(f\"최적의 파라미터로 모델의 정확도 : {gcv_xgb.best_score_}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 최적의 파라미터로 학습되어 있는 모델링 할당\n",
    "estimator_xgb = gcv_xgb.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "실제값과 예측값 정확도 : 0.9950413223140496\n",
      "macro 평균 f1 점수 : 0.995\n"
     ]
    }
   ],
   "source": [
    "# 최적의 모델로 예측해보고 실제값이랑 정확도 비교\n",
    "pred_xgb = estimator_xgb.predict(X_test)\n",
    "print(f\"실제값과 예측값 정확도 : {accuracy_score(y_test, pred_xgb)}\")\n",
    "print(\"macro 평균 f1 점수 : {:.3f}\".format(f1_score(y_test,pred_xgb,average = \"macro\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1375000</td>\n",
       "      <td>804</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0    1  2  3  4  5  6  7  8  9\n",
       "0  1375000  804  0  0  0  0  1  0  0  1"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 나이 -> 20대, monthly_spend, avg_score -> cluster 5 중앙값\n",
    "test = pd.DataFrame([[1375000, 804, 0, 0, 0, 0, 1, 0, 0, 1]] )\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_test = estimator_xgb.predict(test)\n",
    "pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:title={'center':'Feature importance'}, xlabel='F score', ylabel='Features'>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApYAAALGCAYAAAD/fNH9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABIDUlEQVR4nO3dfZzVdZ3//8cLB7nSVRfIbbfUdQmzIPsmVwozHCXNlhRN+7bED6USrM36UkRRCF+VLS9C/FZeoGUXtLuiSwaFQipwUBEiNUoy1CWBtVKQC2G4UIZ5//6Y4zTAoKDvYeYwj/vtdm583p/P+7zP6zMvhed8PnPmREoJSZIk6a1q09wFSJIk6dBgsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0kqQxHx7oiY2Nx1SFJD4e+xlCSIiFXA80BNadftKaX/fJNrXZ9S+kqu2lqqiPgb4PMppW80dy2SWgaDpSRRHyzfn1LalGGtTSmlo9/qOi1dRJwAzEwpvb+ZS5HUQngrXJJeR0T0ioj7I2J+RDwYEe8u7e8bEcWIWBgRj0VE74g4IiKKwBGlYyMiohARMxusd0JELGu4HRHfiIgnIqLTvl6vkbrq1y1t/zIivhcRvyo97wMRcW9ELI2In0ZERWnujyLi66VjiyJiXkQc32DdL5T2Lyyt03MftZ4CTAe6lc71nIjoHBG/iIiHSnM/tcdzJ0XEgoj4fUQMbPCaHym93sKIeKy0ryIiri/V96i3/aUykVLy4cOHj1b/AFYBjwDF0uMk4CjgIeCY0pzewLzSdjfgiNL2PwOzG6y1qcF2gbqreq+NTwCWNdjeDvQpjff5eo3UW79uaXsj8PbS+PvAM8DRpfF9wLml7R8B9wMdSuPhwL2l7X8B7gXalca9gKeBtnvWuue5lMbHACeUtv8WWAdEad6rQL/SsQ8CS0vb/wv4LdC1NO5U+nMc8K+l7QB+AVQ2938nPnz4eP1HRWNhU5JaqY+kBrfCI+KfgX8CfhYRr+0+svTnn4D/HREfAN4DdHmTr/nnlNLS0nb/13m9N7IkpfSX17aBmgbn8hvqwt1rfpRS2l7a/g/gxtL2BcCNKaVXAFJKj0XEi9SF7Oo9at1LSmljRPSLiMsoBW/gb0qH/5hSWlLafqh0ngAXAbeklNaV1tha2v9RYGdE/O/S+OjSOTz8Bl8HSc3IYClJ+1YBFFNKwxo5Nhv4L+pCWQfgzn2sUUPdFb/XtN/j+Jb9fL038soer7mjwXgncNge49e0azD3MKC2kbV3NVLrXiJiAvAO4DvAytLjtYRcX09K6dWIeK2ejnvU85oKYFhK6dnXe01JLYs/YylJ+7YYKEREN4CIOPy1nzkE3g/8e0ppFTBkzydGxGth8o/A+yPiiNL442/y9XL6eINg93+oC8kAPwdGR8Thpdf/X0An6m6rN2Y7f70iCdAHuCel9HvqbnG/fT9quRf4TOkd5kTEUaX9DwD/J0qXbiPi/Q1qltRCecVSkvYhpbQuIj4NTI+I7dRdfbsGeBK4ClgcES8AC/Z46h3Akoj4dkppWkR8B/hVRPwFWPgmXy+n54B7I6ITdT9bennp9adFxNuBhyNiK/Ay8LGU0q4Gt+Yb1vti6Y1Lj1L39ZgMfCcixlF3O37NGxWSUnowIk4E5kXENuquvJ4NTAK+CzwWEdXAC8An3uJ5S2pi/rohSWpFIuJH1L3pZ2YzlyLpEOStcEmSJGVhsJQkSVIW3gqXJElSFl6xlCRJUha+K7wFOProo1O3bt2auwwdgK1bt9KpU6fmLkMHyL6VH3tWnuxb+TmQnj3++OMvpZS6NnbMYNkCHHvssTz22GPNXYYOQLFYpFAoNHcZOkD2rfzYs/Jk38rPgfQsIlbv65i3wiVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWVQ0dwGC7Tt3ccK4e5u7DB2AMT1rGGHPyo59Kz/2rDzZt72tunZwc5dwUHjFUpIkSVkYLCVJkg6iD3zgA8ydO5fa2lpGjx7NaaedRv/+/Vm/fj2vvvoq5557LoVCgYEDB/KnP/0JgOnTp1MoFOjVqxfXXHPNXmvOnDmTyspK+vbty1133XWwT6leWQTLiGgXEf0ajJfsY16j+5tLRBQi4trmrkOSJLUMM2bM4OWXXwbgtttu433vex+LFy9m0aJFdO7cmYqKCu666y6KxSIjR47kxz/+MQDdunWjWCyydOlSZs2axbp16+rX3Lp1K5MnT+bBBx9k/vz5XHvttezYsaNZzq8sgiXwdmB0cxchSZL0Zm3ZsoWf/OQnDBs2DID77ruPp59+mqqqKsaOHUtKiTZt2tCxY0cAnn32WXr27AlAr169AGjTpg2dO3fm8MMPr193yZIlDBo0iHbt2tGpUyf69u3LihUrDvLZ1WmSYBkRJ0TEfRHxvYhYHhGDI+LfI+LXETElIg6LiJsiohgRSyLiK6XnFUrz7omIJyPi/0TEYcB04MyIuL/Ba/xbRDwcEQsjokOD/X8fEQ/sMe+cRmo8LyIejYhHIuKC0r7HIuK2iHgoIn4ZEX9b2j+q9FqLIuLDpX0/ioirIuKBiFgWESeV9leV1v0lMLwpvr6SJKn8fOELX+CKK66gTZu6+LV06VIuuugiHnroIbZv384999wDwLe+9S3e9a538dhjj3HmmWfutsYtt9xCZWUlRx11VP2+tWvX0rVr1/px586d2bhx40E4o7015bvC/wk4HzgCWAWcklJ6LiJ+B/wrsDalVIiINsDMiJhbet7xQKFU27KU0rcj4l+Aa1NK/1Ka8zbgzpTSFRHxHeBsYBZASunPEfFSRHQrve4AYEIj9X0SGJ5SWlmqAeBdwPkppecjYgTw+YiYXlq/CmgL3A/MKc3/c0rprIj4KDAKGANMAc5OKW2IiLFA58a+OBExqvQcunTpysSeNfvxJVVLcWyHunc9qrzYt/Jjz8qTfdvb+PHjqa2tZevWraxatYqOHTtyxBFHsHXrVorFIu985zu599576dy5M71796Z379786le/4sILL2TcuHFs27aNqVOncuqppzJw4ECKxWL92qtXr+aZZ56p37d8+fL6W+f7q7q6+oDm70tTBstlKaVXgQ0RsSKl9Fxp/xrg/cDNACml2ohYAJwErAMeTSntAnZFxOZ9rP1SSun3pe0/AMfscfy7wKeAJ4CfppRSI2uMBi6PiO3UhcFNwIqU0vOl478CKoFTSo8Fpf3HRsRrX7eHGtRwbkS8DfhTSmlDaf9jwIcaO4GU0u3A7QDHndgt3fCkv/mpnIzpWYM9Kz/2rfzYs/Jk3/b23ieX0bFjR6ZOncry5ctZtWoV3bt3p3PnzvTs2ZM5c+YwZMgQTj31VI444ggigq5du/LrX/+aQqHAJZdcwuTJk+nevftea/fo0YMhQ4bQv39/du7cycaNGxk+fDgVFfvfg2KxSKFQeMvn2ZRdbxjmavc4thI4B3iidLWwEvgadT9L2fB5r23vAtrtY729QmNK6dGI+CpwMnDxPupbm1IaGxEfou6K5hjgnyLib0vBcDCwDHgGWJhSuhQgIjqmlGoiouFrJyCADaU1OqWUtgJn7OO1JUlSK3LvvX/9vZ5XXnkl/fr14x//8R8ZNWoUAKeccgrnnXcejz32GKNHj6Zdu3Z06NCBm266CYDZs2ezevXq+jUmTpzIEUccwcqVKxk6dCgjRoxgwIABdOjQgauuuuqAQmVOzfXtxE+AiRHxMHUhcVpK6emIePs+5v8Z6BIRv0wpNXoFsBH3AyellLbs4/iUiHgvdaF1fIPXmRIRJwDPA59OKb0SEWsiYjGwGZhN3RXRvZQC5zeBRRGxlrorppIkSfWuvPLK+u2FCxfudqx3794sWrRor+esX7++0bX69OkDwMiRIxk5cmS+It+kJgmWKaVVwL80GPdrsP2R0uZeZ59SKgLFPZ9XujVeuY/1pja2n7orol96nRo/28jubSmlEY3MvRq4eo99IxpsrwBGlLb/E/jPfb2uJEnSoeqQ+wGIiPg76t7I818ppWdL+0ZQCn4lT6aUPn/wq5MkSTp0HXLBMqX0AtB3j30/An60H8/t90ZzmkKHtofxdCv5DNFDRbFYZNWwQnOXoQNk38qPPStP9q31KpdfkC5JkqQWzmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwqmrsAwfaduzhh3L3NXYYOwJieNYywZ2XHvpUfe1aeDpW+rbp2cHOXUHa8YilJkqQsDJaSJElv4AMf+ABz586tH8+aNYt+/frVj5cuXUpVVRX9+/fn+uuvB2D69OkUCgV69erFNddcs9eaM2fOpLKykr59+3LXXXc1/UkcBN4KlyRJeh0zZszg5Zdfrh/v2rWLadOm1Y937tzJ1VdfzaxZszjmmGPq93fr1o1isUhtbS2nn346l156KV27dgVg69atTJ48mXnz5lFTU8OAAQMYMmQI7du3P3gn1gS8YplRRERz1yBJkvLZsmULP/nJTxg2bFj9vptuumm38Zw5czj++OMZOnQogwYN4oknngCgV69eALRp04bOnTtz+OGH1z9nyZIlDBo0iHbt2tGpUyf69u3LihUrDtJZNZ2yDpYRMT0iFkTEkogYHxFDS/vblvZFRFwfEY9GxM8i4v6IeHcj67SPiLsi4pGIuLe0r3NE3F1af15p3+mlcTEiHoiIE0v7ixExDrg3Ig6LiFtfe15E/ONB/JJIkqSMvvCFL3DFFVfQpk1dZFq+fDmLFy/mox/9aP2cZ599lg0bNjB79mzuuOMOPve5z+22xi233EJlZSVHHXVU/b61a9fWX70E6Ny5Mxs3bmzis2l65X4r/PMppXURcQnwDuATwJ3AR4CfAh8EjkkpnR4RhwO/2sc6JwGvppQGRMRrYftbwA9TSnMa7PsO8OHSa/YGrgcuKh17IqV0bURcCjydUvpsRLwHuAL49J4vGBGjgFEAXbp0ZWLPmrf0hdDBdWyHunc9qrzYt/Jjz8rTodK38ePHU1tby9atW1m1ahUdO3bk61//OldccQXFYpHNmzdTLBZ57rnnOO6443jkkUeAuqucCxYsYPv27UydOpVTTz2VgQMHUiwW69devXo1zzzzTP2+5cuX1986bw7V1dVZXrtsg2VEvA2YGBHVwN8DfwZ2lfYPBT5LXaC7DyCl9GpEPNnYWiml30bE/Ij4LnAvMBc4OaU0p3S8NiK6An9OKa0r7ft1RPxDg2UeLf35AaB3RJxfGq/bx2veDtwOcNyJ3dINT5ZtK1qlMT1rsGflx76VH3tWng6Vvr33yWV07NiRqVOnsnz5cn7/+9/zl7/8hbvvvhuou+q4aNEiLr74YiZPnkyhUODFF1+kS5cunHHGGVxyySVMnjyZ7t2777V2jx49GDJkCP3792fnzp1s3LiR4cOHU1HRPF+3YrFIoVB4y+uUc9eHA4tSSndGxBigK/Bj4PPAlpTS+ohYA1QCP4uIjkC/xhaKiPbAj1JKPyzdDl8C/CUiTk8pPRoRbYGXgHdGROfS2qcCKxss89q3Zs8Ai1NKPymt3TH/qUuSpKZ2771//V2cV155Jf369eOcc86p39evXz/Gjx8PwEknnUT//v2pqKhgypQpAMyePZvVq1fXz584cSJHHHEEK1euZOjQoYwYMYIBAwbQoUMHrrrqqmYLlTmV8xk8CPx7RAwDVlAX7GYDN1B3SxxgBnBeRCwG1gB/BHY0sta7gdtLVz9/n1LaFBFfKu1rB7yUUrowIkYDsyLiVWAT8K+NrHU78MPSLfGt1N1SX5DjhCVJUvO48sor99q3ZMmS+u1JkyYxadKk3Y6vX7++0bX69OkDwMiRIxk5cmS+IluAsg2WKaXfAj0bOXRiw2nAsJRSioijqAt4/9PIWsuAPnvsWwWcvce+hcCARp5faLC9Dfj4fp6GJEnSIaNsg+V+eht1VzXbAG2BcSmlXRFR3GPemJTS4we9OkmSpEPIIR0sU0p/AQY1sr9w8KvZtw5tD+NpP4+0rBSLRVYNKzR3GTpA9q382LPyZN9ar7L+PZaSJElqOQyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScqiorkLEGzfuYsTxt3b3GXoAIzpWcMIewbAqmsHN3cJkqQWwiuWkiRJysJgKUmSpCwMlpLesnXr1jF+/HgmTJgAQG1tLaNHj+a0006jf//+rF+/vtF5AE8//TSDBg2if//+jB07dq+1Z86cSWVlJX379uWuu+46OCckSXpTDJaNiIgPRcRZpe1TIuLoA3z+2ogolh5nNkmRUgsyZswY2rVrx86dOwG47bbbeN/73sfixYtZtGgRnTt3bnQewOjRo7njjjtYtGgRq1at4le/+lX9sa1btzJ58mQefPBB5s+fz7XXXsuOHTsO7slJkvabwbIRKaVfppQeKA2/CPzd/j43Io4EFqWUCqXH/CYpUmpBpk2bRlVVVf34vvvu4+mnn6aqqoqxY8eSUmp0Xk1NDTt27OCEE04A4MILL2Tx4sX1x5csWcKgQYNo164dnTp1om/fvqxYseLgnJQk6YD5rvCSiBgNXFQafh9oD+wEzgHeExHfBkYDfVJKKSIuBSpSSlP3WOpoYON+vN4oYBRAly5dmdizJsNZ6GA5tkPdO8MFxWIRgGXLlrFmzRqKxSKLFi1i8ODBfPjDH+bb3/42V199NQMHDtxr3vr166mtra1f4/nnn+fJJ5+sHxeLRTZv3lw/rq6uZsGCBWzatOlN1VpdXV2/lsqDPStP9q385OqZwRKIiCqgD1CVUqqNiE8BpJTuiIhK4NqU0oqI6AmcCcwDPg6c38hyRwCnRcQjwDPAl1JKm/aclFK6Hbgd4LgTu6UbnrQV5WRMzxrsWZ1Vwwr12y+88AKFQoF3vvOdfOYznwFg+/btPPbYYxQKe8/bvn07t956a/2xtWvX0r59+/rxtm3bePzxx+vHd999N2eddRY9evR4U7UWi8Xd6lDLZ8/Kk30rP7l65q3wOn2AGSml2tK4dh/zbgYuiYhTgd+llLbuOSGl9IeU0skppQHAYuDrTVKx1IIdd9xxPPnkk0DdX1bve9/7Gp3XoUMHXnnlFf70pz8BcM899zBo0KD643369GHu3Lns3LmTbdu2sXz5ct797nc3/QlIkt4UL7nUeQYYDNxTGrdtcGwX0A4gpfQ/EdEGGMs+AmNEVKSUXrtHug54V5NULLVgkydPZtSoUQCccsopnHfeefucO2XKFC666CLatWvHeeedx8knn8zSpUtZuXIlQ4cOZcSIEQwYMIAOHTpw1VVXUVHhX1uS1FL5NzSQUvp5RFRGxBKgGngQ2FQ6PAeYHhETUkozgDuBy1JKf9zHcgMi4hvAq6U1Pt2kxUstRKFQqL+NctJJJ7Fw4cI3nAfQu3fv3d6wA3VXKvv06QPAyJEjGTlyZJPULEnKy2BZklLa+xfo1e2fAcxosOsjwI2vs04R6J+1OEmSpDJgsDwAEfEosDSltKA0PgcY12DKupTSxw503Q5tD+NpP2+5rBSLxd3etCJJkgyWBySldPoe47nA3GYqR5IkqUXxXeGSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwqmrsAwfaduzhh3L3NXYYOwJieNYwog56tunZwc5cgSWpFvGIpSZKkLAyWUiuwbt06xo8fz4QJEwD4j//4Dz74wQ/ygQ98gBtvvLF+3pQpUxg4cCD9+/fnN7/5DdXV1RQKhfrHiSeeyHe+853d1p45cyaVlZX07duXu+6666CelySpZfFWeCMi4kNAbUrpgYg4BVidUtp0gGu0A54DzkwprWiCMqX9NmbMGLp168a2bdsAOO+88xg2bBi1tbWccsopjB49mpdffpmf//znFItFVq5cyRe/+EV+8YtfUCwWAaitreXDH/4wn/rUp+rX3bp1K5MnT2bevHnU1NQwYMAAhgwZQvv27ZvjNCVJzcwrlo1IKf0ypfRAafhF4O/exDKfB7bnq0p686ZNm0ZVVVX9+MgjjwTgxRdf5LjjjiMiOOyww6itreXVV1/lpZdeomvXrrutMX36dAYPHswRRxxRv2/JkiUMGjSIdu3a0alTJ/r27cuKFX4fJUmtlVcsSyJiNHBRafh9oD2wEzgHeE9EfBsYDfRJKaWIuBSoSClNbWStdwDvBx5u+sqlA7du3TqGDBnCc889x2233QbUhc2qqipOPvlkqqurmTdv3m7P+d73vsfs2bN327d27drdAmjnzp3ZuHFj05+AJKlFMlgCEVEF9AGqUkq1EfEpgJTSHRFRCVybUloRET2BM4F5wMeB8xtZqw1wC3VXLK96ndccBYwC6NKlKxN71uQ9KTWpYzvUvTO8pXvtNjbAsmXLWLNmTf2+b37zm2zfvp2JEyeyZcsW1qxZw8qVK7njjjvYsmULF198Md/61reoqKjgqaee4phjjuHXv/71buuvXr2aZ555pn7N5cuX061bt91etyWprq5usbWpcfasPNm38pOrZwbLOn2AGSml2tK4dh/zbga+ERGbgN+llLY2MufrwM9SSqsjYp8vmFK6Hbgd4LgTu6UbnrQV5WRMzxrKoWerhhV2G7/wwgsUCgVefvlljjrqKAB+8IMfcMopp/Dyyy/Tu3dvzjjjDGpqarj66qvp27cvRx55JD/72c/44he/SGVl5W7r9ejRgyFDhtC/f3927tzJxo0bGT58OBUVLfNrUywWKRQKzV2GDoA9K0/2rfzk6lnL/Nv/4HsGGAzcUxq3bXBsF9AOIKX0P6UrkmOpC5CNuQB4tvQGoL5A14gYnlLa0CSVS2/CV77yFf7whz+QUuKCCy6gR48enHjiiXzqU59i4MCBvPLKK1x22WX1P4v56KOPcv3119c/f+nSpaxcuZKhQ4cyYsQIBgwYQIcOHbjqqqtabKiUJDU9/wUAUko/j4jKiFgCVAMPAptKh+cA0yNiQkppBnAncFlK6Y/7WOvU17Yj4kfU3UY3VKrZvfYrg4D6n6tsqGPHjkyfPr3R5+55C7xPnz706dMHgJEjRzJy5Mi8xUqSypLBsiSlNHYf+2cAMxrs+ghwY2NzG3nuiLdemSRJUnkwWB6AiHgUWJpSWlAanwOMazBlXUrpY81SnCRJUjMzWB6AlNLpe4znAnPf6rod2h7G036mc1kpFot7vTFGkqTWzl+QLkmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKoqK5CxBs37mLE8bd29xl6ACM6VnDiGbs2aprBzfba0uStC9esZQkSVIWBkupTK1bt47x48czYcIEAKZPn06hUKBXr15cc8019fO+8pWvMHDgQE477TSWLVu22xqzZs2iX79+e609c+ZMKisr6du3L3fddVeTnock6dDhrfBGRMSHgNqU0gMRcQqwOqW0aT+f+3bgh0AHYCvw8ZTSliYrVq3WmDFj6NatG9u2bQOgW7duFItFamtrOf3007n00kt5/PHH2bFjBwsXLuQPf/gDl19+OfPmzQNg165dTJs2ba91t27dyuTJk5k3bx41NTUMGDCAIUOG0L59+4N6fpKk8uMVy0aklH6ZUnqgNPwi8HcH8PSNwLkppYHAo8C5ueuTAKZNm0ZVVVX9uFevXgC0adOGzp07c/jhh/Pkk09yxhlnAHDyySezadOm+vk33XQTw4YN22vdJUuWMGjQINq1a0enTp3o27cvK1asaNqTkSQdEgyWJRExOiIeKT1GRMRnIuLTwDnAtIgYFhG/jogozb80Ij6z5zoppR0ppZ0RcRhwPPDUwT0TtXa33HILlZWVHHXUUbzvfe9j5syZpJR49tlnWbVqFSklli9fzuLFi/noRz+61/PXrl1L165d68edO3dm48aNB/MUJEllylvhQERUAX2AqpRSbUR8CiCldEdEVALXppRWRERP4ExgHvBx4Px9rHcD8L+BucDv9zFnFDAKoEuXrkzsWZP3pNSkju1Q987w5lIsFgFYtmwZa9asoVgssm3bNqZOncqpp57KwIEDKRaLtGvXjoqKCt7//vfTvXt33vWud/HAAw8wbtw4rrjiCorFIps3b65fD2D16tU888wz9fuWL19ef5u93FVXVx8S59Ga2LPyZN/KT66eGSzr9AFmpJRqS+Pafcy7GfhGRGwCfpdS2trYpJTSGGBM6YrmeODKRubcDtwOcNyJ3dINT9qKcjKmZw3N2bNVwwr12y+88AKFQoFLLrmEyZMn0717993mFgp1c++77z4ef/xxdu7cyZFHHsndd98N1F2hXLRoEePHjwegR48eDBkyhP79+7Nz5042btzI8OHDqago//9Gi8Vi/ddD5cGelSf7Vn5y9az8/6XI4xlgMHBPady2wbFdQDuAlNL/REQbYCzw9cYWioi/SSltLg2fB7o1ScXSHmbPns3q1avrxxMnTuSUU05hyJAhpJTo3r07t956K+3bt2fw4L/+Hsx+/foxfvx4li5dysqVKxk6dCgjRoxgwIABdOjQgauuuuqQCJWSpKbnvxZASunnEVEZEUuAauBBYFPp8BxgekRMSCnNAO4ELksp/XEfyw2MiK8BO4ANlG53S02hUCjUf4e5fv36Ruc88sgjr7vGkiVLAOjTpw99+vQBYOTIkYwcOTJfoZKkVsFgWZJSGruP/TOAGQ12fQS48XXW+QXwi7zVSZIktXwGywMQEY8CS1NKC0rjc4BxDaasSyl9rFmKkyRJamYGywOQUjp9j/Fc6t75/ZZ0aHsYT/vZz2WlWCzu9gYaSZLk77GUJElSJgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlsV/BMiI+W/rz7yNiRkSc17RlSZIkqdzs7xXLfyn9+XnqPiN7dJNUI0mSpLK1v8GyTUScAexKKT0DtG3CmiRJklSG9jdYfpm6z8i+ISLaA79supIkSZJUjvYrWKaUfgX8DPgg8ArwraYsSpIkSeVnf9+8cxNwLjCWutvgP2zKoiRJklR+9vdW+Mkppa8CW1NKrwJva8KaJEmSVIb2N1jWRMQ/ACki/gZo14Q1SZIkqQztb7D8PPB94D3ADOrezCNJkiTVq9jPeR9KKX24SSuRJElSWdvfK5ZVEeHvrpQkSdI+7e8Vy7bAUxHxG2AXkFJKn2i6siRJklRu9jdY/p8mrUKSJEllb3+DZWrSKiRJklT29jdYXkdduAygG3WfvjOgqYqSJElS+dmvYJlSGtpwHBETm6YcSZIklav9fVf4nrpmrUKSJEllb7+uWEbEnfz15yz/Dni0ySqSJElSWdrfn7EcV/ozARtSStVNVI8kSZLK1P7eCr8qpbQ6pbQmpVQdEd9v0qokSZJUdl73imVEnAL8C1CIiG+WdrcHejd1YZIkSSovb3Qr/DlgLnAO8MvSvl3AVU1ZlCRJksrP6wbLlNJmYGFEDE4p/fkg1SRJkqQytL8/Y/nPEfFERPw+Iv4YEUuatCpJkiSVnf0NlpdR90k7M4H+wMKmKkiSJEnlaX+D5csppW3AYSmlvwCnN2FNkiRJKkP7Gyz/PSK6AOsiYi6wrglrkiRJUhna388K/1Fp84aI+GFKaUPTlSRJkqRytF9XLCPif0XELyKiCGyLiIuatixJkiSVm/29FX4jcDGQUko7gE82XUmSJEkqR/sbLEkpbaTus8IB/qZpypEkSVK52t9guSAibgS6RMR4YHkT1iRJkqQytL9v3rkqIgYBfwaeSSl9o2nLkiRJUrl53SuWETGmwbBDSulbKaVZTVyTJEmSytAb3Qof3GD7S01ZiCRJksrbfr95R5IkSXo9b/QzlqdGxKNAAO9psJ1SSn6soyRJkuq9brBMKR11sAqRJElSefNWuCRJkrIwWEqSJCmL/fo9lmpa23fu4oRx9zZ3GToAY3rWMKIZe7bq2sFvPEmSpIPMK5aSJEnKwmApSZKkLAyWUplat24d48ePZ8KECQBMnz6dQqFAr169uOaaa+rnfeUrX2HgwIGcdtppLFu27HXnvmbmzJlUVlbSt29f7rrrroNyPpKk8mewzCwiro+I+RHx64g4s7nr0aFrzJgxtGvXjp07dwLQrVs3isUiS5cuZdasWaxbt465c+eyY8cOFi5cyA9+8APGjBmzz7mv2bp1K5MnT+bBBx9k/vz5XHvttezYsaNZzlGSVF4MlvlNSimdCXwUGNfcxejQNW3aNKqqqurHvXr1AqBNmzZ07tyZww8/nCeffJIzzjgDgJNPPplNmzbtc+5rlixZwqBBg2jXrh2dOnWib9++rFix4iCdlSSpnLW6d4VHxHTgWKAD8AmgNzAGWA+sBRanlKZGxHnAWOrC9/dTSj+MiJHAJ0tLfSmltGTP9VNKW0qb7wJ+9zp1jAJGAXTp0pWJPWtynJ4OkmM71L0zvLkUi0UAli1bxpo1a+rHUHcb+x/+4R/4zW9+Q0qJqVOncvTRR/OnP/2J//7v/2bBggVExF5zG669efPm+jWrq6tZsGBBfSgtZ9XV1bt9rdTy2bPyZN/KT66etbpgCXw+pbQuIi4BhgIfAQamlLZHxM0AEXE0MBo4E6gBHoyIO4FPA2emlLZFRKNXeyPiLOA6oBOwz98Jk1K6Hbgd4LgTu6UbnmyNrShfY3rW0Jw9WzWsUL/9wgsvUCgU2LJlC2PHjuWDH/wgF110EQCFQoFXX32VK6+8klNPPZUBAwZwxhlnNDr3Ndu2bePxxx+nUKh7jbvvvpuzzjqLHj16HKzTazLFYrH+vFQe7Fl5sm/lJ1fPWtWt8Ih4G/B/I+JaYFDpsTSltL005fHSn92pu+L4ALCAuiucxwIjgW9GxDjgcBqRUnogpfQB4GzgP5vqXKQ9XX755XzpS1/aKyheccUVLFy4kA9+8IP069fvdecC9OnTh7lz57Jz5062bdvG8uXLefe7331QzkGSVN5a22Wy4cCilNKdETEG2AH0joi2KaWdwBnAIuA56m5jfySllCKiY+kqZYeU0uiIuIy6kPndhotHRAVweEppG/ASrSy4q3nNnj2b1atX148nTpzIKaecwpAhQ0gp0b17d2699dZ9zj3iiCNYuXIlQ4cOZcSIEQwYMIAOHTpw1VVXUVHR2v6qkCS9Ga3tX4sHgX+PiGHACmAD8FNgaUT8BXge2FG6VT4TWBwRm4FfAROAO0u3yWuAzzayfjvgFw1uk3+tKU9GKhQK9bcu1q9f3+icRx55ZK99+5rbp08fAEaOHMnIkSPzFClJajVaVbBMKf0W6NlwX+lq5Q2lMDgDWFaa+z3ge3s8//w3WH8rdT+XKUmS1Oq0qmC5Dz+OiLdTd7XxnpTSsv19YkQU99g1JqX0eGNzX0+HtofxtJ/9XFaKxeJub6CRJEkGS1JKn3gLzy1kLEWSJKms+eYSSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGVhsJQkSVIWFc1dgGD7zl2cMO7e5i5DB2BMzxpGZOjZqmsHZ6hGkqSWwSuWkiRJysJgKTWzdevWMX78eCZMmFC/78EHH+T9738/O3bsqN+3dOlSqqqq6N+/P9dffz0A06dPp1Ao0KtXL6655pq91p45cyaVlZX07duXu+66q+lPRpLUqnkrvBER8SGgNqX0QEScAqxOKW3az+ceB9wM/A3wCvDxlNLGJitWZW/MmDF069aNbdu2AXVh8Ne//jWHH354/ZydO3dy9dVXM2vWLI455pj6/d26daNYLFJbW8vpp5/OpZdeSteuXQHYunUrkydPZt68edTU1DBgwACGDBlC+/btD+4JSpJaDa9YNiKl9MuU0gOl4ReBvzuAp7cD/r+U0kBgFnBp7vp0aJk2bRpVVVX14/PPP59vfOMbdOzYsX7fnDlzOP744xk6dCiDBg3iiSeeAKBXr14AtGnThs6dO+8WRpcsWcKgQYNo164dnTp1om/fvqxYseIgnZUkqTXyimVJRIwGLioNvw+0B3YC5wDviYhvA6OBPimlFBGXAhUppakN10kpPdtguBFITVy6WoFnn32WDRs2MHv2bJ5//nmGDh3K4sWL64/fcsstVFZWctRRR9XvW7t2bf3VS4DOnTuzcaMXzyVJTcdgCUREFdAHqEop1UbEpwBSSndERCVwbUppRUT0BM4E5gEfB85/nTVPpi6oXryP46OAUQBdunRlYs+ajGekpnZsh7p3hr9VxWIRgGXLlrFmzZr6McCmTZt46KGHOPzww3nuuec47rjjeOSRRwDYsmULCxYsYPv27UydOpVTTz2VgQMH7vb81atX88wzz9TvW758ef2t89aqurq6VZ9/ObJn5cm+lZ9cPTNY1ukDzEgp1ZbGtfuYdzPwjYjYBPwupbS1sUmlYPpe6m6Jb2tsTkrpduB2gONO7JZueNJWlJMxPWvI0bNVwwr12y+88AKFwl/HRx99NFVVVbRv356OHTsyefJkCoUCL774Il26dOGMM87gkksuYfLkyXTv3n2vtXv06MGQIUPo378/O3fuZOPGjQwfPpyKitb731qxWNzta6yWz56VJ/tWfnL1rPX+C7O7Z4DBwD2lcdsGx3ZR93OTpJT+JyLaAGOBrze2UER8GHhHSmlM05Wr1qZPnz6cdNJJ9O/fn4qKCqZMmQLA7NmzWb16df28iRMncsQRR7By5UqGDh3KiBEjGDBgAB06dOCqq65q1aFSktT0/FcGSCn9PCIqI2IJUA08CGwqHZ4DTI+ICSmlGcCdwGUppT/uY7lzgX4RcWZpvDSl9JUmLF+HgEKhsNd3invekpg0aRKTJk3abd/69esbXa9Pnz4AjBw5kpEjR2arU5Kk12OwLEkpjd3H/hnAjAa7PgLc+Drr/Gvm0iRJksqCwfIARMSj1F2BXFAanwOMazBlXUrpY81SnCRJUjMzWB6AlNLpe4znAnPf6rod2h7G035mdFkpFou7vfFGkiT5C9IlSZKUicFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVkYLCVJkpSFwVKSJElZGCwlSZKUhcFSkiRJWRgsJUmSlIXBUpIkSVlUNHcBgu07d3HCuHubu4xD2qprBzd3CZIkHfK8YilJkqQsDJZqVaZMmcLAgQPp378/v/nNb5g+fTqFQoFevXpxzTXX1M+bMGFC/bzf//73u61RXV3NpEmTqKqq4vzzz2fz5s0H+zQkSWqRDJaZRcR1EVGMiMci4pzmrkd/tWnTJn7+859TLBb58Y9/zMSJE+nWrRvFYpGlS5cya9Ys1q1bx8MPP8yLL77IwoULue222xg7duxu69x4442cdtppPPTQQ5x11lnceuutzXRGkiS1LAbL/P4rpVQAPgz8WzPXogYOO+wwamtrefXVV3nppZfo2rUrvXr1AqBNmzZ07tyZww8/nPvvv5+hQ4cC0KNHDzZs2LDbOvPnz6dQKABw4YUXsnjx4oN6HpIktVSt7s07ETEdOBboAHwC6A2MAdYDa4HFKaWpEXEeMJa68P39lNIPI2Ik8MnSUl9KKS3Zc/2U0mOlzc3ApqY8Fx2YI488kqqqKk4++WSqq6uZN29e/bFbbrmFyspKjjrqKNauXUvXrl3rj1VUVFBbW0ubNnXfh73yyitUVNT9r9O5c2c2btx4cE9EkqQWqtUFS+DzKaV1EXEJMBT4CDAwpbQ9Im4GiIijgdHAmUAN8GBE3Al8GjgzpbQtIvZ5tTci2gHfAb75OnNGAaMAunTpysSeNTnOTftQLBZZvHgxK1eu5I477mDLli1cfPHFfOMb32Dq1KmceuqpDBw4kGKxyMsvv8z8+fN56aWXANiyZQsPPfRQ/Vpbtmxh8+bNFIvF+lBZLBab47R0gKqrq+1VmbFn5cm+lZ9cPWtVwTIi3gZMjIhq4O+BdwBLU0rbS1MeBw4HugPvAh4o7e9C3VXOkcA3I+IF4P8BOxp5je7AROD6lNLv9lVLSul24HaA407slm54slW14qBbNazAU089Re/evTnjjDOoqanh6quv5o477mDy5Ml07969fu6WLVt48MEH+cIXvsBTTz3Fe9/73vpb3wBnn302v/3tb5kwYQK33norQ4cO3e24Wq5isWivyow9K0/2rfzk6llr+xnL4cCilNI44LfAfwO9I6Jt6fgZpT+fA34HnFH6eck+KaXVwH+nlEYDG6kLmbuJiA7AFGDU64VKNY8RI0awdOlSBg4cyIABA7jssssoFouMGjWKQqFAoVBg/vz5DB48mFdffZXKykq+/OUvc9111wHw1a9+lVdffZWvfe1rzJ49m0KhwOOPP84nP/nJN3hlSZJah9Z2mexB4N8jYhiwAtgA/BRYGhF/AZ4HdpRulc8EFkfEZuBXwATgztJt8hrgs42s3xP4AHBfRLy276MppQ2NzNVB1rFjR6ZPn77bvpEj9/r+AKDRd3q/FjC7dOnCdddd53fjkiTtoVUFy5TSb6kLf/Uiom1K6YbSz0zOAJaV5n4P+N4ezz//DdZfSt0tdkmSpFanVQXLffhxRLwdaAfck1Jatr9PjIjiHrvGpJQez1ibJElS2Wj1wTKl9Im38NxCjho6tD2Mp/0sa0mSVOZa25t3JEmS1EQMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKoqK5CxBs37mLE8bd29xllLVV1w5u7hIkSWr1vGIpSZKkLAyWOmTcdNNNFAqF+keXLl3qj82aNYt+/frVj5cuXUpVVRX9+/fn+uuv322d6upqhg4dSlVVFeeffz6bN28+aOcgSVI581Z4IyLiQ0BtSumBiDgFWJ1S2nQAzz8O+Drwm5TSbU1UpvZw+eWXc/nllwPw05/+lOeeew6AXbt2MW3atPp5O3fu5Oqrr2bWrFkcc8wxe61z4403cu655/KJT3yCm2++mVtvvZWvfvWrB+ckJEkqY16xbERK6ZcppQdKwy8Cf7e/z42Iw4EbgG1ANEF5egO1tbXcfPPN9SHzpptuYtiwYfXH58yZw/HHH8/QoUMZNGgQTzzxxG7Pnz9/Ph/72McAuPDCC1m8ePHBK16SpDJmsCyJiNER8UjpMSIiPhMRnwbOAaZFxLCI+HVERGn+pRHxmT3XSSm9mlL6GPC7g3wKKpk1axZnnXUW7du3Z/ny5SxevJiPfvSj9cefffZZNmzYwOzZs7njjjv43Oc+t9vzX3nlFdq2bQtA586d2bhx40GtX5KkcuWtcCAiqoA+QFVKqTYiPgWQUrojIiqBa1NKKyKiJ3AmMA/4OHD+W3jNUcAogC5dujKxZ81bPIvWrVgs1m9/61vfYuzYsdx///2MGzeOK664gmKxyObNmykWizz33HMcd9xxPPLIIwBs2bKFBQsWUPqegS1btjB//nzatGlTHyobrg91P4e55z61fPat/Niz8mTfyk+unhks6/QBZqSUakvj2n3Muxn4RkRsAn6XUtr6Zl8wpXQ7cDvAcSd2Szc8aSveilXDCgCsX7+eTp06ccEFF3Dvvfdy5JFHcvfddwOwdu1aFi1axMUXX8zkyZMpFAq8+OKLdOnShTPOOKN+rbPPPpuXX36ZCy64gFtvvZWhQ4dSKBR2e71isbjXPrV89q382LPyZN/KT66emWbqPAMMBu4pjds2OLYLaAeQUvqfiGgDjKXuzTlqYR566CFOO+00AAYPHszgwX/9/Zb9+vVj/PjxAJx00kn079+fiooKpkyZAsBXv/pVJk2axNe+9jWGDx/Ot7/9bbp168bNN9988E9EkqQyZLAEUko/j4jKiFgCVAMPAptKh+cA0yNiQkppBnAncFlK6Y/NU61ezwUXXMAFF1zQ6LElS5bUb0+aNIlJkybtdvy6664DoEuXLsyZM6fpipQk6RBlsCxJKY3dx/4ZwIwGuz4C3Lgf6/0oT2WSJEnlwWB5ACLiUWBpSmlBaXwOMK7BlHWld4RLkiS1OgbLA5BSOn2P8Vxg7ltdt0Pbw3jaz7qWJEllzt9jKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLCqauwDB9p27OGHcvc1dRou36trBzV2CJEl6HV6xlCRJUhYGS5WdpUuXUlVVRf/+/bn++ut5/vnnOeecc6isrOQLX/hC/bwJEyYwcOBA+vfvz+9///vd1qiurmbo0KFUVVVx/vnns3nz5oN9GpIkHXIMlplFRPuI+HRE/KK5azkU7dy5k6uvvppZs2axaNEivvKVrzBlyhS+/OUv8/DDD7NhwwYef/xxHn74YV588UUWLlzIbbfdxtixY3db58Ybb+Tcc8/loYce4qyzzuLWW29tpjOSJOnQYbDM78tAAF2bu5BD0Zw5czj++OMZOnQogwYN4oknnuDII49kw4YN1NbWsmXLFo455hjuv/9+hg4dCkCPHj3YsGHDbuvMnz+fj33sYwBceOGFLF68+KCfiyRJh5pW9+adiJgOHAt0AD4B9AbGAOuBtcDilNLUiDgPGEtd+P5+SumHETES+GRpqS+llJbsuX5K6d9Kr3PpG9QxChgF0KVLVyb2rMlxeoe0YrHI3Llzefrpp/n617/OunXrGD58OFdddRWf+9zn+PKXv8y73/1u1qxZw7JlyzjxxBMpFosAbN26lfnz59OmTd33UuvWrWPRokUA1NTU8Nxzz9XP3R/V1dUHNF8tg30rP/asPNm38pOrZ60uWAKfTymti4hLgKHAR4CBKaXtEXEzQEQcDYwGzgRqgAcj4k7g08CZKaVtEfGWrvamlG4Hbgc47sRu6YYnW2MrDsyqYQV++9vf0rt3bwYNGgTAd7/7Xb73ve/x2GOP8c53vpPvfve7PPXUU5x88sl069aNyspKAI455hjOPPPM+rWOPvpoqqqqaNOmDWvXruVd73oXhUJhv2spFosHNF8tg30rP/asPNm38pOrZ63qVnhEvA34vxFxLTCo9FiaUtpemvJ46c/uwLuAB4AF1F3hPBYYCXwzIsYBhx/M2lXntNNOY86cOQC8+OKLtG3blhdeeIG//du/BeDtb387q1atorKykhkzZgDw1FNP8Y53vGO3dfr27cusWbMA+OlPf8oHP/jBg3gWkiQdmlrbZbLhwKKU0p0RMQbYAfSOiLYppZ3AGcAi4Dngd8BHUkopIjqWrlJ2SCmNjojLqAuZ322uE2mt+vTpw0knnUT//v2pqKhgypQp/PnPf+bss8+mbdu2dOzYkR/96Ed06dKF++67j8rKSo488khuu+02AL761a8yadIkvva1rzF8+HC+/e1v061bN26++eZmPjNJkspfawuWDwL/HhHDgBXABuCnwNKI+AvwPLCjdKt8JrA4IjYDvwImAHeWbpPXAJ9thvoFTJo0iUmTJtWPTz31VM4999y95jX2Tu/rrrsOgC5dutRf+ZQkSXm0qmCZUvot0LPhvtLVyhtKPzM5A1hWmvs94Ht7PP/8A3itfm+1XkmSpHLSqoLlPvw4It4OtAPuSSkt298nRkRxj11jUkqPNzb39XRoexhP+3GFkiSpzLX6YJlS+sRbeG4hYymSJEllrVW9K1ySJElNx2ApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwMlpIkScrCYClJkqQsDJaSJEnKwmApSZKkLAyWkiRJysJgKUmSpCwipdTcNbR6EbEFeLq569AB6QK81NxF6IDZt/Jjz8qTfSs/B9Kz41NKXRs7UJGvHr0FT6eUejV3Edp/EfGYPSs/9q382LPyZN/KT66eeStckiRJWRgsJUmSlIXBsmW4vbkL0AGzZ+XJvpUfe1ae7Fv5ydIz37wjSZKkLLxiKUmSpCwMlpIkScrCYNmMImJSRCyMiEUR8d7mrkcQEV0j4hsRMak0Piki5pV69K0G8/bq3b7mqmlFxNERMT0iihHxUET8o31r+SLi8Ij4RalvCyPiH+xb+YiIJyLiHHtWHiLiydL/a8WI+EST9i2l5KMZHkAlcHtpuwdwX3PX5CMBTAMmAteWxnOAE0rb/wX03VfvGpvb3OfTGh7A3wN/X9oeDNxs31r+g7oLGx1L2/8f8HX7Vh4P4CJgJXCOPSuPB/DgHuMm65tXLJvP2cCdACml5cDfNm85AkgpXQw8BBARFUD7lNKq0uGfAqfRSO9eZ66aWErpzymlP5eGG4FXsG8tXkqpNqW0rTR8F/Ak9q3Fi4gjgeHAf1D3ISv2rDzUvrbR1P+2GSybz9uAdQ3GNRFhP1qWrsD6BuP1wDE00jvg2H3M1UESEf8AfBm4AftWFiJibEQ8C/QCnsC+lYPvAP9GXVA5EnvW4kVEJ+CfSj8qdDfwdpqwb36kY/N5md2bU5tSqt3XZDWLTcDRDcbHUPc/XQf26B2wYR9zdRBExEeAc4GRwDbsW1lIKX0L+FZEfBiYgn1r0SJiGLAmpfTriBiMf0eWhZTSVuCfACLiLJr4/zWvkDWfh6n7ORUi4j3A881bjvaUUtoOtCtdCQP4KDCPRnr3OnPVxCLifcC5KaXLUkrr7Vt5iIgjIyJKwzXAYdi3lu4TwHsiYjp1Pfkq8F571rJFxGENhuuARBP+v+YVy+ZzL/DPEfEwsAW4rJnrUeO+BMyIiFeAn6eU/hART9N47/aa2zwltzrnAJURUSyN12DfysG7gf9X+rpvBy4HumDfWqyU0uDXtiPiSmAJdbdG7VnL1i0ifgC8Wnp8FuhME/XNT96RJElSFt4KlyRJUhYGS0mSJGVhsJQkSVIWBktJkiRlYbCUJElSFv66IUlqoSJiM3WfSANwb+kXiktSi2WwlKSW66mUUqG5i5Ck/eWtcEkqUxHxfyPi0YhYEhHHRZ3rImJhaf9JEfH2iJgZEQsiYnFEfKj03Csj4vqIeDgijo+IUaXtRaWPWJSkA+YVS0lqud7T4BOFpqSUfr7H8Y8C708ppdLHIw4HSCkNBIiINsC00nMfioijgWJE3F96fk1KqTIiTgLOBqqAtsD9wJwmPC9JhyiDpSS1XG90K/xy4DsRsQK4FegD/OC1gyml2oj4p5TSQ6XxpohYTd1HJwI8WvrzlNJjQWl8bERUpJRq8p2KpNbAW+GSVL4eSyl9HngHMBh4hrrPTgcgIiqA/4mI/qXxUcDbgJdKU14Ljs8AC1NKhVKQPdVQKenN8IqlJJWh0m3ueRHxCrANmELdLezbI2JRad8oYAxwW0R0pC5IfrF067x+rZTSsohYExGLgc3AbOC7B/WEJB0SIqXU3DVIkiTpEOCtcEmSJGVhsJQkSVIWBktJkiRlYbCUJElSFgZLSZIkZWGwlCRJUhYGS0mSJGXx/wPLH7CydENOCwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 720x864 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,12))\n",
    "plot_importance(estimator_xgb, ax=ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LightGBM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LightGBM과 XGBoost를 비교했을 때의 장점\n",
    "- 빠른 학습 및 예측 수행 시간\n",
    "- 더 작은 메모리 사용량\n",
    "- 카테코리형 피처의 자동 변환 및 최적 분할\n",
    "\n",
    "그러나, 10,000건 이하의 데이터셋에 대해 과적합 문제 발생하기 쉽다는 단점"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>monthly_spend</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>city_1</th>\n",
       "      <th>city_2</th>\n",
       "      <th>city_3</th>\n",
       "      <th>city_4</th>\n",
       "      <th>city_5</th>\n",
       "      <th>ages_1</th>\n",
       "      <th>ages_2</th>\n",
       "      <th>ages_3</th>\n",
       "      <th>city</th>\n",
       "      <th>ages</th>\n",
       "      <th>cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>727000</td>\n",
       "      <td>740</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>707000</td>\n",
       "      <td>782</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>712000</td>\n",
       "      <td>782</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>771000</td>\n",
       "      <td>782</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>738000</td>\n",
       "      <td>781</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   monthly_spend  avg_score  city_1  city_2  city_3  city_4  city_5  ages_1  \\\n",
       "0         727000        740       0       0       0       0       1       0   \n",
       "1         707000        782       0       0       0       0       1       0   \n",
       "2         712000        782       0       0       0       0       1       0   \n",
       "3         771000        782       0       0       0       0       1       0   \n",
       "4         738000        781       0       0       0       0       1       0   \n",
       "\n",
       "   ages_2  ages_3 city ages  cluster  \n",
       "0       0       1   서울  20대        2  \n",
       "1       0       1   서울  20대        2  \n",
       "2       0       1   서울  20대        2  \n",
       "3       0       1   서울  20대        2  \n",
       "4       0       1   서울  20대        2  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_lgbm = df_encoding.copy()\n",
    "df_lgbm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2419, 10) (605, 10)\n"
     ]
    }
   ],
   "source": [
    "# 피쳐와 라벨 분류\n",
    "X_features = df_lgbm.iloc[:, :-3]\n",
    "y_labels = df_lgbm.iloc[:, -1]\n",
    "\n",
    "# train, test 데이터셋 분리\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_features, y_labels, test_size=0.2, random_state=121)\n",
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbm = LGBMClassifier()\n",
    "param_grid_lgbm={'boosting_type' :['gbdt'], # 부스팅 방법 rf, gbdt(디폴트), dart, goos\n",
    "                 'silent':[True],\n",
    "                 'max_depth':[4, 5],\n",
    "                 'min_child_weight':[3,5],\n",
    "                 #'gamma':[0,1],\n",
    "                 'nthread':[4],\n",
    "                 'colsample_bytree':[0.7,0.8],\n",
    "                 #'colsample_bylevel':[0.9],\n",
    "                 'objective':['multiclass'],\n",
    "                 'random_state':[121],\n",
    "                 'learning_rate' : [0.05, 0.1],\n",
    "                 'n_estimators' : [300, 500],\n",
    "                 'subsample' : [0.7, 1.0],\n",
    "                 'metric':['multi_logloss', 'multi_error']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] num_threads is set with n_jobs=-1, nthread=4 will be ignored. Current value: num_threads=-1\n"
     ]
    }
   ],
   "source": [
    "kf = StratifiedKFold(n_splits=5, random_state=121, shuffle=True)\n",
    "\n",
    "gcv_lgbm = GridSearchCV(lgbm, param_grid=param_grid_lgbm, cv = kf, scoring='f1_macro', n_jobs=4, refit = True, return_train_score = True)\n",
    "gcv_lgbm.fit(X_train, y_train)\n",
    "\n",
    "# 각 파라미터값들에 대한 모델 결과값들이 cv_results_ 객체에 할당됨\n",
    "scores_df_lgbm = pd.DataFrame(gcv_lgbm.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "최적의 파라미터 : {'boosting_type': 'gbdt', 'colsample_bytree': 0.7, 'learning_rate': 0.05, 'max_depth': 5, 'metric': 'multi_logloss', 'min_child_weight': 3, 'n_estimators': 300, 'nthread': 4, 'objective': 'multiclass', 'random_state': 121, 'silent': True, 'subsample': 0.7}\n",
      "최적의 파라미터로 모델의 정확도 : 0.9906480129441059\n"
     ]
    }
   ],
   "source": [
    "# 최적의 파라미터는 best_params_에 할당되어있음\n",
    "print(f\"최적의 파라미터 : {gcv_lgbm.best_params_}\")\n",
    "print(f\"최적의 파라미터로 모델의 정확도 : {gcv_lgbm.best_score_}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 최적의 파라미터로 학습되어 있는 모델링 할당\n",
    "estimator_lgbm = gcv_lgbm.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "실제값과 예측값 정확도 : 0.9884297520661157\n",
      "macro 평균 f1 점수 : 0.988\n"
     ]
    }
   ],
   "source": [
    "# 최적의 모델로 예측해보고 실제값이랑 정확도 비교\n",
    "pred_lgbm = estimator_lgbm.predict(X_test)\n",
    "print(f\"실제값과 예측값 정확도 : {accuracy_score(y_test, pred_lgbm)}\")\n",
    "print(\"macro 평균 f1 점수 : {:.3f}\".format(f1_score(y_test,pred_lgbm,average = \"macro\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1375000</td>\n",
       "      <td>804</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0    1  2  3  4  5  6  7  8  9\n",
       "0  1375000  804  0  0  0  0  1  0  0  1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 나이 -> 20대, monthly_spend, avg_score -> cluster 5 중앙값\n",
    "test = pd.DataFrame([[1375000, 804, 0, 0, 0, 0, 1, 0, 0, 1]] )\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5], dtype=int64)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 최적의 모델로 예측해보고 실제값이랑 정확도 비교\n",
    "pred_test = estimator_lgbm.predict(test)\n",
    "pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Feature Importance')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvgAAAFzCAYAAABLvX1lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAitElEQVR4nO3de7RdZX3u8e9jgsGgBuWiES85tXjBQ2XUEJUiBJSCDVpt5VAvCFaM2HNoo5Q2px51tFaLOlC00mMDtOJlhHGMV0BAQEDkYtxYILVFtBjlZgWBgFwUkt/5Y810LLd7JzvZa2ft/eb7GWONzMs73/mba7LCs9/9rplUFZIkSZLa8KhhFyBJkiRpcAz4kiRJUkMM+JIkSVJDDPiSJElSQwz4kiRJUkMM+JIkSVJDDPiSJElSQwz4kjTDJFmb5JtJLu1er5tEXx8cZG3jnGNBkmun+jwTqOM9SXYadh2SNNXiP3QlSTNLkrXAPlV1zwD6uqeqdp5sP5s5xwLgS1W1z1SeZwJ1rGVA75skTWeO4EtSI5IsTPK1JF9PclGS53TbX9iN9F+WZCTJvkkem+RS4LHdvmOSLE7ypb7+/mvkfeNykvcl+U6SncY73wTqvCfJXya5JMm/JlmU5JNJLk9ydZJndO2OSXJGkjOTfKOr/eC+fl7W9XFpd9wf9u1bm2Rp95uOlyU5B3gycE6S5V2bj3f7Vyf5xKhjj0tyXpIbkvxZ377nJjmnO+81Sfbpth/V9xuVs5I8dotvoCQNyOxhFyBJ2irnJHmkW34r8BPgw8DvV9XdSfYFTgVeCvwMOLyqfp7k94D3VNXhwOJuBH8xQJLFmznns4GlVfXOJPM2cb7NmQdcX1UfSPIG4BLgwKoaSfIXwJ8CJ3RtXw68sKpuTvJs4GtJngs8qTvf4qq6PckuwOVJvltVN3THzqmq/bvli7oR/MP7RvBPr6prkwS4OsnCqhrp9u1UVS9P8kTgxiSfBB4Bvgi8oav1UcCcJC8GDgNeWlXrk5wIvB147wTeC0kaOAO+JM1M/UGVLrg/E/hiL68C8Ljuz1uB/5Hkt4G9gF238py3VdXqbvl3NnG+zXmoqs7rlq8Gbu4L1v8CHNfX9pyquhmgqr6X5AfAc4AXAp+rqtu7fT9Lsgp4GbAx4H9pc4UkeWfX31OBPYCNdXy26/euJDcCT6f3G4B/2VhrVW0AHkzyamAf4OLuvdgRuGaC74UkDZwBX5LaMBu4tKpeP8a+c4DPAR8BHgOsHKePR4Ad+tZ3HLX/vgmeb3N+MeqcD/WtPwzMGrXebyfg512bDWP0vb5v+b4x9gOQ5EDgffR+U/Bx4B+A9DXpr+mX3fnmjlEP9N6LU6rqtPHOJ0nbknPwJakNV9GbcvObAEkenWTvbt8+wGeqai3w+6MPTLIx1N8E7NM3f/zIrTzfIG2cJkOS/egF/P8ALgCOTPLkbt8TgFcC543XEfAg8PhueSFweVV9i15AP3ACtVwBHNhNFSLJrO6pPBcCx25835LsnuRpW3aZkjQ4juBLUgOq6o4kbwbOSvIgvdHovwPWAH8NXJXkJ/Tmu/c7g978849W1aeSfAz4VpLbgcu28nyDtBo4rZtj/zBwRFWtB76f5B3A55M8TG/k/oTuh5jx/CNwbpJPAZ8GViW5HLgF+PbmCqmqO5O8Hji9m4qzATiuqs7rfrj5ZpJ19Eb837aV1ytJk+ZjMiVJ01KSY+g91nLZkEuRpBnFKTqSJElSQwz4kiRJUkOcoiNJkiQ1xBF8SZIkqSEGfEmSJKkhPiZzwHbddddasGDBsMuQJElSw6655po7q2q3sfYZ8AdswYIFjIyMbL6hJEmStJWS/Gi8fU7RkSRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGjJ72AW0Zs2t61iw/NxhlyFJkqQptPakJcMuYVyO4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeCPI0mGXYMkSZK0pWYPu4CNkpwFPAl4DHA2cFNVrUyyA3A58GLgA8D+wH8COwF/WlU3jOpnR+BMYA9gXVUtSbIL8H+B3YANVfXSJPsB7wMCPAy8tapuSnIpcD5wQJJXAB8HngNsAI6tqh9O5fsgSZIkTca0CfjA8VV1R5KjgacCrwNWAocDnwdeBjyhqvZL8mjgW+P082zgl1W1f5KNv6H4EPDPVXVe37aPAS/vzrkv8EHgNd2+71TVSUmOBb5XVW9Lshfwf4A3jz5hkqXAUoBZj99tUm+CJEmSNBnTIuAn2R14d5KfA08BbgPWd9tfC7yNXrD+KkBV/TLJmrH6qqrrknw9yd8D59IbjX9uVZ3X7d+QZDfgtqq6o9v27SR79HVzZffnbwP7JnlVt37HOOdcAawAmDN/z9qa90CSJEkahOkyB/8o4IqqWg5c1207EzgeuK+qfgb8GHgJQJK5wIvG6qibovPJqjoe+D9JdgZu76bk0E35uRN4Wjd1hyQvAP6jr5tHuj9vBD5WVYurajFw9GAuV5IkSZoa02IEH7gI+EyS1wM30AvY5wAn05uqA7AKeGWSq+iF/ZuAh8bo6znAiu63Ad+tqnuSvKPbNge4s6r+MMky4MtJfgncA/zJGH2tAP65m6pzP72pPpcM4oIlSZKkqZCqmTGjJMksel+QrSTz6AXtfatq/ZBL+xVz5u9Z848+ZdhlSJIkaQqtPWnJUM+f5JqqWjjWvukygj8Ru9Mb5X8UsAOwvKrWd0+96XdCVV2zzauTJEmSpoEZE/Cr6nbgpWNsX7ztq5EkSZKmp+nyJVtJkiRJA2DAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhoyY56iM1Psvcc8Rob8XFRJkiRtvxzBlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIa4lN0BmzNretYsPzcYZchSZKkcaxt/ImHjuBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ3Z5gE/yZwkL+pbv3qcdmNuH5Yki5OcNOw6JEmSpE0Zxgj+fGDZEM4rSZIkNW+zAT/JgiRfTXJakn9NsiTJZ5J8O8mHk8xK8vEklya5OslfdMct7tp9IcmaJH+WZBZwFnBwkq/1neNvk1ye5LIkj+nb/pQkF45qd9gYNb4yyZVJvpnk1d22kST/mOQbSS5I8sRu+9LuXFckeXm37ZNJ/jrJhUmuTfLsbvsBXb8XAEdt5XssSZIkbTMTHcF/JvA/gQOAlcC7qmpf4GXAnwA/rarFwH7A/kl+qzvuGcARwELguKpaD/wR8PWq+t2uze7Ayqp6CXAdsHE7VXUbcGeS30wyG9gfuGCM+t4EHFVV+wNf7rbtCby3qjbWfHwX3H+3u46DgL/s6+O2qjoE+Btgabftw8DhVXUocMN4b073Q8NIkpH1D6wbr5kkSZI05SYa8K+tql9W1V3ADVX1w277j4F9gHMAqmoDcAnw7G7/lVW1vqp+Adw7Tt93VtV3u+V/B54wav/fA38MvAr4fFXVGH0sA45L8jfA47ttN1TVLd3yt4CnA8/vXpcAXwOe1P3gAPCNvhqemGR34NbumgFGxqmfqlpRVQurauGsufPGayZJkiRNuYkG/P5QvWHUvv8ADgNI8ijgJcD1Yxy3cXk9MGec/n4tvFfVlcDzgNcDnxynvp9W1YnAFcC7um3P3DgtB1gCXAvcCFxWVYu73zi8oKoeGXXuAgLc1fWxU7f9oHHOLUmSJE0bszffZLM+Dbw7yeX0wvqnqup7SeaP0/42YNckF3RTXybia8Czq+q+cfZ/OMnz6P3w8M6+83w4yQLgFuDNVfWLJD9OchW93yicQ+83BL+mqh5J8n7giiQ/Bb4zwVolSZKkocnYM16mlyRnA++oqu9vwTFXV9WLNt9ysObM37PmH33Ktj6tJEmSJmjtSUuGXcKkJbmmqhaOtW8QI/hTJsmT6X1p9nMbw32SY4Bj+pqtqarjt311kiRJ0vQzrQN+Vf0EeOGobZ9k/Ln4/e22+ei9JEmSNGzD+IeuJEmSJE0RA74kSZLUEAO+JEmS1BADviRJktQQA74kSZLUkGn9FJ2ZaO895jHSwLNVJUmSNDM5gi9JkiQ1xIAvSZIkNcSAL0mSJDXEgC9JkiQ1xIAvSZIkNcSn6AzYmlvXsWD5ucMuQ5pW1vpkKUmSthlH8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhjQf8JMcmuSQbvn5SXbewuN/muTS7nXwlBQpSZIkDUjz/5JtVV3Qt/p24CTgnokcm+RxwBVV9eopKE2SJEkauCYDfpJlwGu61dOBHYGHgcOAvZJ8FFgGLKqqSnIsMLuqPjGqq52Bu7dFzZIkSdIgNDdFJ8kBwCLggKran+4aq+oM4HzgjVX1WeBiYOOUmyOBT4/R3WOBFyf5ZpJ/Gm96T5KlSUaSjKx/YN1gL0iSJEnaAs0FfHrhflVVbejWN4zT7lTg6CQvAK6vqvtHN6iqf6+q53Y/KFwF/NVYHVXViqpaWFULZ82dN4BLkCRJkrZOiwH/RuDQvvUd+pbXA3MAqupmetd/Ir2w/2uS9E9humOwZUqSJEmD11zAr6qvAPcmuTrJRcAufbvPA85KsnF+/kpgblXdNE53+ye5IsklwNH0vqArSZIkTVtNfsm2qk4cZ/sqYFXfpsOBj2yin0uB3xlocZIkSdIUajLgT0SSK4HVVXVJt34YsLyvyR1VdcRQipMkSZK20nYb8Ktqv1Hr59N7yo4kSZI0YzU3B1+SJEnanhnwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIZst0/RmSp77zGPkZOWDLsMSZIkbaccwZckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGuJTdAZsza3rWLD83GGXIQ3cWp8OJUnSjOAIviRJktQQA74kSZLUEAO+JEmS1BADviRJktQQA74kSZLUEAO+JEmS1BADviRJktQQA74kSZLUEAP+JiT5YJKvJ/l2koOHXY8kSZK0Of5Ltpv23qq6L8nTgDOArw+7IEmSJGlTZvQIfpKzklyS5Ookv5HkyCSrk5yX5Mwkx3XtXpnk8iRXJHlTt+0tSa7sXi8aq/+quq9b3BO4fttclSRJkrT1ZvoI/vFVdUeSo4HXAocDB1bVg0lOBUiyM7AMOBh4BLgoyUrgzcDBVfVAkjF/0ElyCPABYCdgyXhFJFkKLAWY9fjdBnRpkiRJ0pabsQE/ye7Au5P8HHgK8FRgdVU92DW5Bng08Cx6I/AXdtt3BZ4EvAV4f5KfAKcAD40+R1VdCFyY5BnA54BFY9VSVSuAFQBz5u9Zg7g+SZIkaWvM5Ck6RwFXVNVy4DrgB8C+SXbo9h/U/flDetNrDqqqxcCiqvoR8IOqWgbcTS/s/4oks5PM7VbvZGa/V5IkSdpOzNgRfOAi4DNJXg/cANwFfB5YneR24BbgoW4Kz5eAq5LcC3wLeBewspu+8wjwtjH6nwOc3Td9539P5cVIkiRJgzBjA35VXQfs3b8tyQ5VdXIXylcB13ZtTwNOG3X8qzbT//305u1LkiRJM8aMDfjjODPJfHqj71+oqmsnemCSS0dtOqGqrhlgbZIkSdKUayrgV9XrJnHs4gGWIkmSJA2FXxyVJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhrS1FN0poO995jHyElLhl2GJEmStlOO4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDfEpOgO25tZ1LFh+7rDLkDZprU96kiSpWY7gS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDWk+4Cc5NMkh3fLzk+y8Bcc+PcnZSS5L8rUkT5iyQiVJkqQBaD7gV9UFVXVht/p24MlbcPgc4A1VdSDwZeDYQdcnSZIkDdLsYRcwFZIsA17TrZ4O7Ag8DBwG7JXko8AyYFFVVZJjgdlV9Yn+fqrq+32rdwM1xaVLkiRJk9LcCH6SA4BFwAFVtT/dNVbVGcD5wBur6rPAxcDB3WFHAp/eRJ/PpfcDw6fG2b80yUiSkfUPrBvYtUiSJElbqsUR/EXAqqra0K1vGKfdqcD7ktwDXF9V94/VKMkfA8+jN1XngbHaVNUKYAXAnPl7OsovSZKkoWluBB+4ETi0b32HvuX19ObVU1U307v+E+mF/V+T5OXAU6vqhPHCvSRJkjSdNBfwq+orwL1Jrk5yEbBL3+7zgLOSbJyfvxKYW1U3jdPdK4BXJbm0e31w6iqXJEmSJq/FKTpU1YnjbF8FrOrbdDjwkU308ycDLk2SJEmaUk0G/IlIciWwuqou6dYPA5b3Nbmjqo4YSnGSJEnSVtpuA35V7Tdq/Xx6T9mRJEmSZqzm5uBLkiRJ2zMDviRJktQQA74kSZLUEAO+JEmS1BADviRJktSQ7fYpOlNl7z3mMXLSkmGXIUmSpO2UI/iSJElSQwz4kiRJUkMM+JIkSVJDDPiSJElSQwz4kiRJUkN8is6Arbl1HQuWnzvsMqQxrfUJT5IkNc8RfEmSJKkhBnxJkiSpIQZ8SZIkqSEGfEmSJKkhBnxJkiSpIQZ8SZIkqSEGfEmSJKkhBnxJkiSpIc0H/CSHJjmkW35+kp23oo85SW5L8pyBFyhJkiQNUPMBv6ouqKoLu9W3A0/eim6OBx4cXFWSJEnS1Jg97AKmQpJlwGu61dOBHYGHgcOAvZJ8FFgGLKqqSnIsMLuqPjFGX08F9gEun/rKJUmSpMlpbgQ/yQHAIuCAqtqf7hqr6gzgfOCNVfVZ4GLg4O6wI4FPj9HXo4B/AN65mXMuTTKSZGT9A+sGdi2SJEnSlmou4NML96uqakO3vmGcdqcCRyd5AXB9Vd0/Rpu/Ar5YVT/a1AmrakVVLayqhbPmztvqwiVJkqTJajHg3wgc2re+Q9/yemAOQFXdTO/6T6QX9sfyauDQJGcBBwInJ3niwCuWJEmSBqS5gF9VXwHuTXJ1kouAXfp2nweclWTj/PyVwNyqummcvl5QVX9UVX8EXAacUFV3TWX9kiRJ0mQ0+SXbqjpxnO2rgFV9mw4HPjLBPo+ZfGWSJEnS1Goy4E9EkiuB1VV1Sbd+GLC8r8kdVXXEUIqTJEmSttJ2G/Crar9R6+fTe8qOJEmSNGM1NwdfkiRJ2p4Z8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSGbLdP0Zkqe+8xj5GTlgy7DEmSJG2nHMGXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhriU3QGbM2t61iw/Nxhl6GGrfUpTZIkaRMcwZckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGtJ8wE9yaJJDuuXnJ9l5C46dn+T8JJcl+WqSx01ZoZIkSdIANB/wq+qCqrqwW3078OQtOPxu4BVVdSBwJfCKQdcnSZIkDdLsYRcwFZIsA17TrZ4O7Ag8DBwG7JXko8AyYFFVVZJjgdlV9Yn+fqrqoa6/WcAzgHO2yQVIkiRJW6m5EfwkBwCLgAOqan+6a6yqM4DzgTdW1WeBi4GDu8OOBD49Tn8nA2u71e+O02ZpkpEkI+sfWDeoS5EkSZK2WHMBn164X1VVG7r1DeO0OxU4OskLgOur6v6xGlXVCVX1NOAa4J3jtFlRVQurauGsufMmWb4kSZK09VoM+DcCh/at79C3vB6YA1BVN9O7/hPphf1fk+Txfau3AI8daKWSJEnSgDUX8KvqK8C9Sa5OchGwS9/u84Czkmycn78SmFtVN43T3YFJrkzydeAY4P1TVbckSZI0CE1+ybaqThxn+ypgVd+mw4GPbKKfs4GzB1udJEmSNHWaDPgTkeRKYHVVXdKtHwYs72tyR1UdMZTiJEmSpK203Qb8qtpv1Pr59J6yI0mSJM1Yzc3BlyRJkrZnBnxJkiSpIQZ8SZIkqSEGfEmSJKkhBnxJkiSpIdvtU3Smyt57zGPkpCXDLkOSJEnbKUfwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSG+BSdAVtz6zoWLD932GVoM9b6pCNJktQoR/AlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIYY8DchyQeSXJpkJMlhw65HkiRJ2hwD/qZ9rqoWAy8H/nbItUiSJEmbNaMDfpKzklyS5Ookv5HkyCSrk5yX5Mwkx3XtXpnk8iRXJHlTt+0tSa7sXi8aq/+qGukW7wXu2SYXJUmSJE3C7GEXMEnHV9UdSY4GXgscDhxYVQ8mORUgyc7AMuBg4BHgoiQrgTcDB1fVA0nG/UEnyRzgY8D7N9FmKbAUYNbjdxvEdUmSJElbZcYG/CS7A+9O8nPgKcBTgdVV9WDX5Brg0cCzgD2BC7vtuwJPAt4CvD/JT4BTgIfGOMezgHcDH6yq68erpapWACsA5szfsyZ9cZIkSdJWmslTdI4Crqiq5cB1wA+AfZPs0O0/qPvzh8D1wEHdfPpFVfUj4AdVtQy4m17Y/xVJHgN8GFi6qXAvSZIkTSczdgQfuAj4TJLXAzcAdwGfB1YnuR24BXiom8LzJeCqJPcC3wLeBazspu88ArxtjP73Bn4b+GqSjdv+oKrumrpLkiRJkiZnxgb8qrqOXgj/L0l2qKqTuzn1q4Bru7anAaeNOv5Vm+l/Nb2pP5IkSdKMMWMD/jjOTDIfmAN8oaquneiBSS4dtemEqrpmgLVJkiRJU66pgF9Vr5vEsYsHWIokSZI0FDP5S7aSJEmSRjHgS5IkSQ0x4EuSJEkNMeBLkiRJDTHgS5IkSQ1p6ik608Hee8xj5KQlwy5DkiRJ2ylH8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIb4mMwBW3PrOhYsP3fYZcxIa328qCRJ0qQ5gi9JkiQ1xIAvSZIkNcSAL0mSJDXEgC9JkiQ1xIAvSZIkNcSAL0mSJDXEgC9JkiQ1xIAvSZIkNcSAvwlJdkzy5iRnD7sWSZIkaSIM+Jv250CA3YZdiCRJkjQRMzrgJzkrySVJrk7yG0mOTLI6yXlJzkxyXNfulUkuT3JFkjd1296S5Mru9aKx+q+qv62q07flNUmSJEmTMXvYBUzS8VV1R5KjgdcChwMHVtWDSU4FSLIzsAw4GHgEuCjJSuDNwMFV9UCSSf2gk2QpsBRg1uMd7JckSdLwzNiAn2R34N1Jfg48BXgqsLqqHuyaXAM8GngWsCdwYbd9V+BJwFuA9yf5CXAK8NDW1lJVK4AVAHPm71lb248kSZI0WTN5is5RwBVVtRy4DvgBsG+SHbr9B3V//hC4HjioqhYDi6rqR8APqmoZcDe9sC9JkiTNeDN2BB+4CPhMktcDNwB3AZ8HVie5HbgFeKibwvMl4Kok9wLfAt4FrOym7zwCvG0I9UuSJEkDN2MDflVdB+zdvy3JDlV1cjenfhVwbdf2NOC0Uce/agvONeaXcCVJkqTpZsYG/HGcmWQ+MAf4QlVdO9EDk1w6atMJVXXNAGuTJEmSplxTAb+qXjeJYxcPsBRJkiRpKGbyl2wlSZIkjWLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhrS1FN0poO995jHyElLhl2GJEmStlOO4EuSJEkNMeBLkiRJDTHgS5IkSQ0x4EuSJEkNMeBLkiRJDfEpOgO25tZ1LFh+7rDLmJHW+vQhSZKkSXMEX5IkSWqIAV+SJElqiAFfkiRJaogBX5IkSWqIAV+SJElqiAFfkiRJaogBX5IkSWqIAV+SJElqSPMBP8mhSQ7plp+fZOctPP7pST6R5K1TUqAkSZI0QM0H/Kq6oKou7FbfDjx5oscmeTRwMvAAkCkoT5IkSRqo2cMuYCokWQa8pls9HdgReBg4DNgryUeBZcCiqqokxwKzq+oT/f1U1S+BI5Ic0/UhSZIkTWvNjeAnOQBYBBxQVfvTXWNVnQGcD7yxqj4LXAwc3B12JPDpSZxzaZKRJCPrH1g3qfolSZKkyWgu4NML96uqakO3vmGcdqcCRyd5AXB9Vd2/tSesqhVVtbCqFs6aO29ru5EkSZImrcWAfyNwaN/6Dn3L64E5AFV1M73rP5Fe2JckSZJmvOYCflV9Bbg3ydVJLgJ26dt9HnBWko3z81cCc6vqpm1dpyRJkjQVmvySbVWdOM72VcCqvk2HAx+ZQH+fHExlkiRJ0tRqMuBPRJIrgdVVdUm3fhiwvK/JHVV1xFCKkyRJkrbSdhvwq2q/Uevn03vKjiRJkjRjNTcHX5IkSdqeGfAlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhmy3T9GZKnvvMY+Rk5YMuwxJkiRtpxzBlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIaYsCXJEmSGmLAlyRJkhpiwJckSZIakqoadg1NSXIf8L1h1yEAdgXuHHYRArwX04n3Ynrxfkwf3ovpw3sxMc+oqt3G2jF7W1eyHfheVS0cdhGCJCPei+nBezF9eC+mF+/H9OG9mD68F5PnFB1JkiSpIQZ8SZIkqSEG/MFbMewC9F+8F9OH92L68F5ML96P6cN7MX14LybJL9lKkiRJDXEEX5IkSWqIAX9Akrw3yWVJrkjyvGHXs71IsibJpd3rdUmeneTi7j58qK+d92cKJNktyfuSvLdbn/D7P15bbZ0x7sVRSf6t+2x8ra+d92KKJdk5yVnde/+NJP/Nz8ZwjHMv/GwMSZJHJzm7e+8vS7KHn40pUlW+JvkCXgKs6Jb/O/DVYde0vbyAi0atnwcs6JY/B7zQ+zOl7/+ngHcDJ23p+z9W22Ffz0x+jXEvjgd+f1Qb78W2uRdPAZ7SLS8BTvWzMa3uhZ+N4d2PRwFzu+U3AH/lZ2NqXo7gD8bvAisBqupfgScOt5ztyoaNC0lmAztW1dpu0+eBF+P9mTJV9UbgG7Bl7/8m2mor9d+Lzs7A3aOaeS+2gaq6rapu61bvBn6Bn42hGONe3I+fjaGpqg1V9UC3uiewBj8bU8KAPxi7A3f0rT+SxPd2iiXZCXhm92vX/wfMB37W1+RnwBPw/mwruzHB9x940jhtNTizgQ8muTzJ0m6b92IbSrIH8OfAyfjZGKq+e3EKfjaGKsmJSb4PLAS+g5+NKeG/ZDsY6/jV/8g2VNWG8RprMKrqfuCZAEkOAT5Mb2RmoyfQ+wviMXh/toV7mOD7D9w1TlsNSFW9B3hPkrnAl5NcwRh/V+G9mBJJDgdeAbwFeAA/G0PTfy+q6meAn40hqqoPAR9K8nK24P/beD+2iKOYg3E58BqAJHsBtwy3nO1Dkll9q3cABczpRmoA/gC4GO/PNlFVDzLB938TbTUg3a+zAR4E7qP3+fBebANJfgt4RVW9tap+5mdjeEbfi26bn40hSfK4JOlWfwzMws/GlHAEfzDOBX4vyeX0/rJ465Dr2V78ZpJ/An7Zvd4G7AKsSvIL4CtV9e9Jvof3Z1t5BxN//3+t7XBKbtbfJVlE7+/5L1bVvyW5Ae/FtnAY8JIkl3brP8bPxrCMdS/+08/G0DwHOKV7Px8E/hewK342Bs5/6EqSJElqiFN0JEmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSGGPAlSZKkhhjwJUmSpIYY8CVJkqSG/H8grd8JYGZSEgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_importance = estimator_lgbm.feature_importances_\n",
    "sorted_idx = np.argsort(feature_importance)\n",
    "fig = plt.figure(figsize=(12, 6))\n",
    "plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')\n",
    "plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])\n",
    "plt.title('Feature Importance')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Catboost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파라미터들이 최적화가 잘 되어있어서 튜닝에 크게 신경쓰지 않아도 됨"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>monthly_spend</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>city</th>\n",
       "      <th>ages</th>\n",
       "      <th>cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>727000</td>\n",
       "      <td>740</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>707000</td>\n",
       "      <td>782</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>712000</td>\n",
       "      <td>782</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>771000</td>\n",
       "      <td>782</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>738000</td>\n",
       "      <td>781</td>\n",
       "      <td>서울</td>\n",
       "      <td>20대</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   monthly_spend  avg_score city ages  cluster\n",
       "0         727000        740   서울  20대        2\n",
       "1         707000        782   서울  20대        2\n",
       "2         712000        782   서울  20대        2\n",
       "3         771000        782   서울  20대        2\n",
       "4         738000        781   서울  20대        2"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cb = df.copy()\n",
    "df_cb = df_cb[['monthly_spend', 'avg_score', 'city', 'ages', 'cluster']]\n",
    "df_cb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cb.city.fillna('없음', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 피쳐와 라벨 분류\n",
    "X_features = df_cb.iloc[:, :-1]\n",
    "y_labels = df_cb.iloc[:, -1]\n",
    "\n",
    "# train, test 데이터셋 분리\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_features, y_labels, test_size=0.2, random_state=121)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid_cb = {'iterations':[100, 500],\n",
    "          'depth': [4, 5, 6, 7],\n",
    "          'learning_rate':[0.05, 0.1],\n",
    "          'logging_level' : ['Silent'],\n",
    "          'l2_leaf_reg': [5]}\n",
    "\n",
    "cb = CatBoostClassifier(loss_function='MultiClass', random_seed=121)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = StratifiedKFold(n_splits=5, random_state=121, shuffle=True)\n",
    "\n",
    "gcv_cb = GridSearchCV(cb, param_grid=param_grid_cb, cv = kf, scoring='f1_macro', n_jobs=4, refit = True, return_train_score = True)\n",
    "\n",
    "gcv_cb.fit(X_train, y_train, cat_features=['city', 'ages'])\n",
    "\n",
    "# 각 파라미터값들에 대한 모델 결과값들이 cv_results_ 객체에 할당됨\n",
    "scores_df_cb = pd.DataFrame(gcv_cb.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "최적의 파라미터 : {'depth': 6, 'iterations': 500, 'l2_leaf_reg': 5, 'learning_rate': 0.1, 'logging_level': 'Silent'}\n",
      "최적의 파라미터로 모델의 정확도 : 0.9971856906198194\n"
     ]
    }
   ],
   "source": [
    "# 최적의 파라미터는 best_params_에 할당되어있음\n",
    "print(f\"최적의 파라미터 : {gcv_cb.best_params_}\")\n",
    "print(f\"최적의 파라미터로 모델의 정확도 : {gcv_cb.best_score_}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 최적의 파라미터로 학습되어 있는 모델링 할당\n",
    "estimator_cb = gcv_cb.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "실제값과 예측값 정확도 : 0.9950413223140496\n",
      "macro 평균 f1 점수 : 0.995\n"
     ]
    }
   ],
   "source": [
    "# 최적의 모델로 예측해보고 실제값이랑 정확도 비교\n",
    "pred_cb = estimator_cb.predict(X_test)\n",
    "print(f\"실제값과 예측값 정확도 : {accuracy_score(y_test, pred_cb)}\")\n",
    "print(\"macro 평균 f1 점수 : {:.3f}\".format(f1_score(y_test,pred_cb,average = \"macro\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1375000, 804, '서울', '20대']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 나이 -> 20대, monthly_spend, avg_score -> cluster 5 중앙값\n",
    "test = [1375000, 804, '서울', '20대']\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2], dtype=int64)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 최적의 모델로 예측해보고 실제값이랑 정확도 비교\n",
    "pred_test = estimator_cb.predict(test)\n",
    "pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Feature Importance')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvgAAAFzCAYAAABLvX1lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAX4klEQVR4nO3de5BmdX3n8c9XBkdBBRVQxMTZVRYvRULFwVtQRrxhBo3ZmHUjIZhVEbNLQnSNVBnNJsZkjCUhRncT0A3eFmrFxEQIQVFABBEHg6IVvE9QwAgoiIogM9/9o89stZNpZ4DueYZfv15VXfM855znnG8/x5F3nz7dU90dAABgDPeY9QAAAMDiEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAdzNVtaGqPl5V508fL7wL+/rTxZxtgWOsqqrLl/o42zHH71fV7rOeA2CplX/oCuDupao2JDmou29chH3d2N173tX9bOMYq5J8oLsPWsrjbMccG7JI7xvAzswVfIBBVNXqqvpQVX20qs6tqkdOyx8/Xem/oKrWV9XBVXWfqjo/yX2mdS+qqjVV9YF5+/v/V943P66qN1TVp6tq94WOtx1z3lhVr66q86rqc1X1uKo6taourKpLquph03Yvqqp3VNU7q+pj0+yHzdvP06d9nD+97pfnrdtQVcdM3+l4elWdmeTBSc6sqhOmbd46rb+0qv5yi9ceW1VnV9WVVfXb89Y9qqrOnI57WVUdNC0/at53VE6vqvvc4RMIsEhWzHoAAO6UM6vq9unxy5J8M8mJSX6xu79TVQcneVuSpyW5IckR3f29qvqFJL/f3UckWTNdwV+TJFW1ZhvHPCDJMd39mqra4yccb1v2SPLZ7n5jVf1akvOSHNrd66vqd5P8VpJXTts+O8nju/vrVXVAkg9V1aOSPGg63pruvraqHpjkwqr6fHdfOb12ZXcfMj0+d7qCf8S8K/hv7+7Lq6qSXFJVq7t7/bRu9+5+dlU9IMkXq+rUJLcn+dskvzbNeo8kK6vqiUkOT/K07t5YVa9K8jtJXr8d7wXAohP4AHdP80M1U7g/PMnfzvVqkuS+059XJ/lPVfVzSR6dZK87ecxruvvS6fHP/4TjbcsPu/vs6fElSb4+L6z/Kcmx87Y9s7u/niTd/YWq+nKSRyZ5fJL3dfe107obquqMJE9PsjnwP7CtQarqNdP+HppkvySb53jvtN9vV9UXk/x05r4D8E+bZ+3uTUluqapfSnJQko9M78W9kly2ne8FwKIT+ABjWJHk/O4+civrzkzyviR/luTeSU5bYB+3J9l13vN7bbH+5u083rbcusUxfzjv+Y+S7LLF8/l2T/K9aZtNW9n3xnmPb97K+iRJVR2a5A2Z+07BW5P8zyQ1b5P5M902HW+3rcyTzL0XJ3X3KQsdD2BHcg8+wBg+kblbbh6RJFV1z6o6cFp3UJL3dPeGJL+45QuranPUfzXJQfPuH3/BnTzeYtp8m0yq6kmZC/yvJDknyQuq6sHTuvsneW6SsxfaUZJbktxverw6yYXd/cnMBfqh2zHLRUkOnW4VSlXtMv1Wng8necnm962q9qmqn7pjnybA4nEFH2AA3X1dVb04yelVdUvmrkb/SZIrkvxBkk9U1Tczd7/7fO/I3P3nf97d76qqtyT5ZFVdm+SCO3m8xXRpklOme+x/lORXuntjki9V1SuSvL+qfpS5K/evnL6IWchfJTmrqt6V5N1JzqiqC5N8I8mntjVId19fVUcmeft0K86mJMd299nTFzcfr6qbMnfF/+V38vMFuMv8mkwAdkpV9aLM/VrL42c8CsDdilt0AABgIAIfAAAG4hYdAAAYiCv4AAAwEIEPAAAD8WsyF9lee+3Vq1atmvUYAAAM7LLLLru+u/fe2jqBv8hWrVqV9evXb3tDAAC4k6rqXxZa5xYdAAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgayY9QCjueLqm7LqhLNmPQYAAEtow7q1sx5hQa7gAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADGRZBH5VnV5V51XVJVX176vqBVV1aVWdXVXvrKpjp+2eW1UXVtVFVfUb07KXVtXF08cTZvuZAADAT7Zi1gPsIMd193VVdXSSX01yRJJDu/uWqnpbklTVnkmOT3JYktuTnFtVpyV5cZLDuvsHVbXVL4iq6pgkxyTJLvfbe6k/FwAAWNDwgV9V+yR5XVV9L8lDkjw0yaXdfcu0yWVJ7pnkPyTZP8mHp+V7JXlQkpcm+eOq+maSk5L8cMtjdPfJSU5OkpX77t9L9skAAMA2LIdbdI5KclF3n5DkM0m+nOTgqtp1Wv/U6c+vJflskqd295okj+vuf0ny5e4+Psl3Mhf7AACw0xr+Cn6Sc5O8p6qOTHJlkm8neX+SS6vq2iTfSPLD6RaeDyT5RFV9N8knk7w2yWnT7Tu3J3n5DOYHAIDtNnzgd/dnkhw4f1lV7drdb57uqT8jyeXTtqckOWWL1z9vx0wKAAB33fCBv4B3VtW+SVYm+ZvuvnzG8wAAwKJYloHf3S+c9QwAALAUlsMP2QIAwLIh8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgK2Y9wGgO3G+PrF+3dtZjAACwTLmCDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwkBWzHmA0V1x9U1adcNasxwCWoQ3r1s56BAB2Aq7gAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4CepqmdV1TOmxz9bVXvOeCQAALhTBH6S7j6nuz88Pf2dJA+e5TwAAHBnrZj1ALNSVccnef709O1J7pXkR0kOT/LoqvrzJMcneVx3d1W9JMmK7v7LGYwLAADbZVlewa+qpyR5XJKndPchmd6H7n5Hkn9M8uvd/d4kH0ly2PSyFyR59wL7O6aq1lfV+o0/uGnJ5wcAgIUsy8DPXNyf0d2bpuebFtjubUmOrqrHJvlsd39/axt198ndvbq7V++y2x5LMC4AAGyf5Rr4X0zyrHnPd533eGOSlUnS3V/P3Hv0qszFPgAA7NSWZeB3998n+W5VXVJV5yZ54LzVZyc5vao2359/WpLduvurO3pOAAC4o5btD9l296sWWH5GkjPmLToiyZ/tkKEAAOAuWpZX8LdXVV2c5NbuPm/WswAAwPZYtlfwt0d3P2nWMwAAwB3hCj4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBABD4AAAxE4AMAwEAEPgAADETgAwDAQAQ+AAAMROADAMBAVsx6gNEcuN8eWb9u7azHAABgmXIFHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGsmLWA4zmiqtvyqoTzpr1GMAWNqxbO+sRAGCHcAUfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAaywwO/qlZW1RPmPb9kge22unxWqmpNVa2b9RwAAPCTzOIK/r5Jjp/BcQEAYHjbDPyqWlVV/1BVp1TV56pqbVW9p6o+VVUnVtUuVfXWqjq/qi6pqt+dXrdm2u5vquqKqvrtqtolyelJDquqD807xh9V1YVVdUFV3Xve8odU1Ye32O7wrcz43Kq6uKo+XlW/NC1bX1V/VVUfq6pzquoB0/JjpmNdVFXPnpadWlV/UFUfrqrLq+qAaflTpv2ek+SoO/keAwDADrO9V/AfnuS/JnlKktOSvLa7D07y9CS/meRb3b0myZOSHFJVPzO97mFJfiXJ6iTHdvfGJP85yUe7+5nTNvskOa27n5zkM0k2L093X5Pk+qp6RFWtSHJIknO2Mt9vJDmquw9J8nfTsv2TvL67N8983BTuz5w+j6cmefW8fVzT3c9I8odJjpmWnZjkiO5+VpIrF3pzpi8a1lfV+o0/uGmhzQAAYMltb+Bf3t23dfe3k1zZ3V+bll+V5KAkZyZJd29Kcl6SA6b1F3f3xu6+Ncl3F9j39d39+enxPye5/xbr/yLJf0nyvCTv7+7eyj6OT3JsVf1hkvtNy67s7m9Mjz+Z5KeT/Oz0cV6SDyV50PSFQ5J8bN4MD6iqfZJcPX3OSbJ+gfnT3Sd39+ruXr3LbnsstBkAACy57Q38+VG9aYt1X0lyeJJU1T2SPDnJZ7fyus2PNyZZucD+/k28d/fFSR6T5Mgkpy4w37e6+1VJLkry2mnZwzfflpNkbZLLk3wxyQXdvWb6jsNju/v2LY7dSSrJt6d97D4tf+oCxwYAgJ3Gim1vsk3vTvK6qrowc7H+ru7+QlXtu8D21yTZq6rOmW592R4fSnJAd9+8wPoTq+oxmfvi4TXzjnNiVa1K8o0kL+7uW6vqqqr6ROa+o3Bm5r5D8G909+1V9cdJLqqqbyX59HbOCgAAM1Nbv+Nl51JVH0zyiu7+0h14zSXd/YRtb7m4Vu67f+979Ek7+rDANmxYt3bWIwDAoqmqy7p79dbWLcYV/CVTVQ/O3A/Nvm9z3FfVi5K8aN5mV3T3cTt+OgAA2Pns1IHf3d9M8vgtlp2ahe/Fn7/dDr96DwAAszaLf+gKAABYIgIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABiLwAQBgIAIfAAAGIvABAGAgAh8AAAYi8AEAYCACHwAABrJi1gOM5sD99sj6dWtnPQYAAMuUK/gAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMJAVsx5gNFdcfVNWnXDWrMdY1jasWzvrEQAAZsYVfAAAGIjABwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYiMAHAICBCPwFVFXNegYAALijVsx6gM2q6vQkD0py7yQfTPLV7j6tqnZNcmGSJyZ5Y5JDkvxrkt2T/FZ3X7nFfu6V5J1J9ktyU3evraoHJvlfSfZOsqm7n1ZVT0ryhiSV5EdJXtbdX62q85P8Y5KnVNVzkrw1ySOTbEryku7+2lK+DwAAcFfsNIGf5Ljuvq6qjk7y0CQvTHJakiOSvD/J05Pcv7ufVFX3TPLJBfZzQJLbuvuQqtr8HYo3Jfnr7j573rK3JHn2dMyDk/xpkudP6z7d3euq6iVJvtDdL6+qRyf5vSQv3vKAVXVMkmOSZJf77X2X3gQAALgrdorAr6p9kryuqr6X5CFJrkmycVr+q0lenrmw/ock6e7bquqKre2ruz9TVR+tqr9IclbmrsY/qrvPntZvqqq9k1zT3ddNyz5VVfvN283F058/l+Tgqnre9Py6BY55cpKTk2Tlvvv3nXkPAABgMews9+AfleSi7j4hyWemZe9MclySm7v7hiRXJXlyklTVbkmesLUdTbfonNrdxyX5varaM8m10y05mW75uT7JT0237qSqHpvkK/N2c/v05xeTvKW713T3miRHL86nCwAAS2OnuIKf5Nwk76mqI5NcmbnAPjPJmzN3q06SnJHkuVX1iczF/leT/HAr+3pkkpOn7wZ8vrtvrKpXTMtWJrm+u3+5qo5P8ndVdVuSG5P85lb2dXKSv55u1fl+5m71OW8xPmEAAFgK1X33uKOkqnbJ3A/IdlXtkbnQPri7N854tB+zct/9e9+jT5r1GMvahnVrZz0CAMCSqqrLunv11tbtLFfwt8c+mbvKf48kuyY5obs3Tr/1Zr5XdvdlO3w6AADYCdxtAr+7r03ytK0sX7PjpwEAgJ3TzvJDtgAAwCIQ+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADCQFbMeYDQH7rdH1q9bO+sxAABYplzBBwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYiMAHAICBCHwAABiIwAcAgIEIfAAAGIjABwCAgQh8AAAYSHX3rGcYSlXdnOQLs56DHWqvJNfPegh2OOd9+XHOlyfnffm5u5zzh3X33ltbsWJHT7IMfKG7V896CHacqlrvnC8/zvvy45wvT8778jPCOXeLDgAADETgAwDAQAT+4jt51gOwwznny5Pzvvw458uT87783O3PuR+yBQCAgbiCDwAAAxH4i6SqXl9VF1TVRVX1mFnPw9Kpqr2r6g1V9frp+QFV9ZHp3L9p1vOx+Kpqz6o6varOr6qPVdW/c97HVlX3rKoPTuf8gqrazzlfPqrq01V1uHO+PFTVFdPf9fOr6oUjnHeBvwiq6slJHtTdhyZ5WZK75f8Y2G5vTnJrkl2n5ycleXF3/3ySVVX1+FkNxpLZLckruntNkjcm+e9x3kd3e5IXTOf8lCRHxzlfFqrq+Un2mJ6eFOd8OfjX7l4zffyfDHDeBf7ieGaS05Kkuz+X5AGzHYel1N2/nuRjSVJVK5Lcq7s3TKvfn+SJMxqNJdLd13T3NdPT72TuCzznfWDdvam7fzA93T/JFXHOh1dV901yVJL3Zu7fCnLOl4dNmx+M8t91gb849kly3bznt1eV93Z52DvJDfOe35Dk/jOahSVWVftl7ur9m+O8D6+qXlVVX0qyOsmn45wvB29J8keZC777xjkfXlXtnuTh0+2X/zfJvhngvPuXbBfHTfnxk7+puzcttDFDuTHJnvOe3z8//sUeg6iqI5I8J8lLk/wgzvvwuvtNSd5UVc9OcmKc86FV1ZFJruruT1XV2vj/92Whu7+f5OFJUlXPyCB/111lXhwXJnl+klTVo5N8Y7bjsKN09y1JVk5XdpPkPyb5yAxHYglU1c8keU53v6y7b3Dex1dV962qmp5elWSXOOeje2GSR1fV6Zn7b/qrkzzGOR9bVe0y7+l1SToD/F13BX9xnJXkF6rqwiQ3Z+4HbVk+XpHkjKq6Ncnfd/c/z3ogFt3hSZ5cVedPz6+K8z66RyY5aTq/tyT5b0n2inM+rO5eu/lxVf2PJJdk7vYM53xsj6iq/53ktunj5UkemLv5efcPXQEAwEDcogMAAAMR+AAAMBCBDwAAAxH4AAAwEIEPAAADEfgAADAQgQ8AAAMR+AAAMJD/BwCDcZUhqwKPAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_importance = estimator_cb.feature_importances_\n",
    "sorted_idx = np.argsort(feature_importance)\n",
    "fig = plt.figure(figsize=(12, 6))\n",
    "plt.barh(range(len(sorted_idx)), feature_importance[sorted_idx], align='center')\n",
    "plt.yticks(range(len(sorted_idx)), np.array(X_test.columns)[sorted_idx])\n",
    "plt.title('Feature Importance')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBoost와 CatBoost가 accuracy, f1 score 모두 0.995 정도로 성능은 비슷\n",
    "\n",
    "나이는 20대지만, 나머지 모두 cluster 5의 모습을 보이는 test 값(특수한 경우)에 대해\n",
    "\n",
    "XGBoost는 cluster 5로, CatBoost는 cluster 2로 분류\n",
    "\n",
    "test가 cluster5로 분류되었으면 좋겠음\n",
    "\n",
    "-> XGBoost를 최종 모델로 선정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 모델 저장"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 모델 저장\n",
    "estimator_xgb.save_model('xgb_model.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_xgb_model = XGBClassifier() # 모델 초기화\n",
    "new_xgb_model.load_model('xgb_model.model') # 모델 불러오기"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "cb5285e2cda14a4805400a189e0eabfa02913f397385f5c221a1fc076b50ea98"
  },
  "kernelspec": {
   "display_name": "Python 3.7.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}