stacking.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "주희 - stacking.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "6E9L7Gp6HcxM",
        "eGvB-9C1dT-F",
        "9XsyleuEe_H0",
        "Fh9-A5kki8pL",
        "vE2LIzwUWU7F",
        "a2Vqd61JSdLQ",
        "MbxuS9XGfi0-",
        "D2lgJxOJhB_8",
        "yvfhyUwNiRMP",
        "-7BaH2GTi2tV",
        "7_xlENgJjmZ_",
        "VxxA4BtOWqwR",
        "QTGrQTG0_DcA"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_3LcHWdoaKXK"
      },
      "source": [
        "#**EDA를 통한 변수 추가 모델**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pxp2JcmbNN_l",
        "outputId": "682203d8-fb4f-4e0a-d722-c37568452c0f"
      },
      "source": [
        "# 내 드라이브에 대한 주소\r\n",
        "from google.colab import drive\r\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xH9XAIwVNM6i"
      },
      "source": [
        "import numpy as np\r\n",
        "import pandas as pd\r\n",
        "import tensorflow as tf\r\n",
        "from tensorflow import keras\r\n",
        "import seaborn as sns\r\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tR74GZMCBBuz"
      },
      "source": [
        "# 내 드라이브에 대한 주소\r\n",
        "path = \"/content/drive/MyDrive/Colab Notebooks/dacon_news\"\r\n",
        "\r\n",
        "train = pd.read_csv(path + \"/data/news_train.csv\")\r\n",
        "#try_train = pd.read_csv(\"try_title_data.csv\")\r\n",
        "test = pd.read_csv(path + \"/data/news_test.csv\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        },
        "id": "VH0uB590RJZ5",
        "outputId": "e86c49fc-ea67-451e-b4a3-574bec451ed9"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>n_id</th>\n",
              "      <th>date</th>\n",
              "      <th>title</th>\n",
              "      <th>content</th>\n",
              "      <th>ord</th>\n",
              "      <th>info</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>[이데일리 MARKETPOINT]15:32 현재 코스닥 기관 678억 순매도</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>\"실적기반\" 저가에 매집해야 할 8월 급등유망주 TOP 5 전격공개</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>하이스탁론, 선취수수료 없는 월 0.4% 최저금리 상품 출시</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>종합 경제정보 미디어 이데일리 - 무단전재 &amp; 재배포 금지</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NEWS09727</td>\n",
              "      <td>20200626</td>\n",
              "      <td>롯데·공영 등 7개 TV 홈쇼핑들, 동행세일 동참</td>\n",
              "      <td>전국적인 소비 붐 조성에 기여할 예정</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        n_id      date  ... ord info\n",
              "0  NEWS02580  20200605  ...   1    0\n",
              "1  NEWS02580  20200605  ...   2    1\n",
              "2  NEWS02580  20200605  ...   3    1\n",
              "3  NEWS02580  20200605  ...   4    0\n",
              "4  NEWS09727  20200626  ...   1    0\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o346VrtzZatW"
      },
      "source": [
        "# content 맨앞이 [이거나 (이면 0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcwUcZBRVEvd"
      },
      "source": [
        "train[\"content_startswith_[\"]=train.content.apply(lambda x : str(x).startswith(\"[\" ) or str(x).startswith(\"(\") or str(x).startswith(\"제목\"))+0"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upG2RgbFWdeb"
      },
      "source": [
        "# 타이틀을 이용한 feature\r\n",
        "- 해당 title에 몇가지 단어가 들어갈 경우 약 90% 이상이 info가 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKfe60qER799"
      },
      "source": [
        "title_noise = ['적중 100%', '글로벌 주요 뉴스', '[전문가 의견]', \r\n",
        "               '[포커스]', '※','■', '▶' ,'◆','★',' TOP', 'BEST',\r\n",
        "'전문가의 눈', '전문가선정', '전문가의견','】','후속주도 감사합니다',\r\n",
        "               '전문가추천', '주요이슈']\r\n",
        "\r\n",
        "def title_choose(x):\r\n",
        "  if (\"종목\" in x[-6:]) or (\"관련주\" in x[-5:]):\r\n",
        "    return 1\r\n",
        "  for noise in title_noise:\r\n",
        "    if noise in x.upper():\r\n",
        "        return 1\r\n",
        "  return 0"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1scNCdG8ZWRy"
      },
      "source": [
        "train[\"info1_title\"]=train['title'].apply(title_choose)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JP_h-HL7RmvU"
      },
      "source": [
        "# content를 이용한 feature"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NlrmrsR1RqCo"
      },
      "source": [
        "content_noise = ['00%', '긴급공개', '긴급 공개','임상3상', 'TOP','BEST','# ','대장株','대장주','카톡','원\"만']\r\n",
        "\r\n",
        "def content_choose(x):\r\n",
        "  if (x.count(\"..\")>=2) or (x.count(\"+\")>=2) or (x.count(\"~\")>=2) :\r\n",
        "    return 1\r\n",
        "  elif (x=='관련기사') or (x==\"관련 테마분석\") or (x==\"코스피\") or (x==\"코스닥\"):\r\n",
        "    return 1\r\n",
        "  for noise in content_noise:\r\n",
        "    if noise in x.upper():\r\n",
        "        return 1\r\n",
        "  return 0"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YWoQmRopTf79"
      },
      "source": [
        "train[\"info1_content\"]=train[\"content\"].apply(content_choose)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vbS7TN-xgARn"
      },
      "source": [
        "# Order을 이용한 feature"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P5Tf50Emf-39"
      },
      "source": [
        "title_group = (train.groupby([\"title\"]).count())[\"n_id\"]\r\n",
        "train[\"new_ord\"]=train.apply(lambda x: x[\"ord\"]/title_group[x[\"title\"]], axis=1)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "SWM9950gXs8b",
        "outputId": "6cba6caf-7c5f-45b0-d55c-49866873c533"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>n_id</th>\n",
              "      <th>date</th>\n",
              "      <th>title</th>\n",
              "      <th>content</th>\n",
              "      <th>ord</th>\n",
              "      <th>info</th>\n",
              "      <th>content_startswith_[</th>\n",
              "      <th>info1_title</th>\n",
              "      <th>info1_content</th>\n",
              "      <th>new_ord</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>[이데일리 MARKETPOINT]15:32 현재 코스닥 기관 678억 순매도</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.250000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>\"실적기반\" 저가에 매집해야 할 8월 급등유망주 TOP 5 전격공개</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>하이스탁론, 선취수수료 없는 월 0.4% 최저금리 상품 출시</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.750000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>종합 경제정보 미디어 이데일리 - 무단전재 &amp; 재배포 금지</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NEWS09727</td>\n",
              "      <td>20200626</td>\n",
              "      <td>롯데·공영 등 7개 TV 홈쇼핑들, 동행세일 동참</td>\n",
              "      <td>전국적인 소비 붐 조성에 기여할 예정</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.066667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        n_id      date  ... info1_content   new_ord\n",
              "0  NEWS02580  20200605  ...             0  0.250000\n",
              "1  NEWS02580  20200605  ...             1  0.500000\n",
              "2  NEWS02580  20200605  ...             0  0.750000\n",
              "3  NEWS02580  20200605  ...             0  1.000000\n",
              "4  NEWS09727  20200626  ...             0  0.066667\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0TOb5PyeZc40"
      },
      "source": [
        "# tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FwAQw1o2Av2C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a822b16e-c982-4718-d5ac-e9d947eb7e4e"
      },
      "source": [
        "# konlpy Mecab 사용하기\r\n",
        "\r\n",
        "!set -x \\\r\n",
        "&& pip install konlpy \\\r\n",
        "&& curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh | bash -x"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+ pip install konlpy\n",
            "Collecting konlpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/85/0e/f385566fec837c0b83f216b2da65db9997b35dd675e107752005b7d392b1/konlpy-0.5.2-py2.py3-none-any.whl (19.4MB)\n",
            "\u001b[K     |████████████████████████████████| 19.4MB 1.3MB/s \n",
            "\u001b[?25hCollecting tweepy>=3.7.0\n",
            "  Downloading https://files.pythonhosted.org/packages/67/c3/6bed87f3b1e5ed2f34bd58bf7978e308c86e255193916be76e5a5ce5dfca/tweepy-3.10.0-py2.py3-none-any.whl\n",
            "Collecting colorama\n",
            "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
            "Collecting beautifulsoup4==4.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/d4/10f46e5cfac773e22707237bfcd51bbffeaf0a576b0a847ec7ab15bd7ace/beautifulsoup4-4.6.0-py3-none-any.whl (86kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 9.3MB/s \n",
            "\u001b[?25hCollecting JPype1>=0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b7/21/9e2c0dbf9df856e6392a1aec1d18006c60b175aa4e31d351e8278a8a63c0/JPype1-1.2.0-cp36-cp36m-manylinux2010_x86_64.whl (453kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 42.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.6/dist-packages (from konlpy) (1.19.4)\n",
            "Requirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.6/dist-packages (from konlpy) (4.2.6)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tweepy>=3.7.0->konlpy) (1.3.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tweepy>=3.7.0->konlpy) (1.15.0)\n",
            "Requirement already satisfied: requests[socks]>=2.11.1 in /usr/local/lib/python3.6/dist-packages (from tweepy>=3.7.0->konlpy) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from JPype1>=0.7.0->konlpy) (3.7.4.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->tweepy>=3.7.0->konlpy) (3.1.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (2020.12.5)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6; extra == \"socks\" in /usr/local/lib/python3.6/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (1.7.1)\n",
            "Installing collected packages: tweepy, colorama, beautifulsoup4, JPype1, konlpy\n",
            "  Found existing installation: tweepy 3.6.0\n",
            "    Uninstalling tweepy-3.6.0:\n",
            "      Successfully uninstalled tweepy-3.6.0\n",
            "  Found existing installation: beautifulsoup4 4.6.3\n",
            "    Uninstalling beautifulsoup4-4.6.3:\n",
            "      Successfully uninstalled beautifulsoup4-4.6.3\n",
            "Successfully installed JPype1-1.2.0 beautifulsoup4-4.6.0 colorama-0.4.4 konlpy-0.5.2 tweepy-3.10.0\n",
            "+ curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh\n",
            "+ bash -x\n",
            "+ mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic\n",
            "+ set -e\n",
            "++ uname\n",
            "+ os=Linux\n",
            "+ [[ ! Linux == \\L\\i\\n\\u\\x ]]\n",
            "+ hash sudo\n",
            "+ sudo=sudo\n",
            "+ python=python3\n",
            "+ hash pyenv\n",
            "+ at_user_site=\n",
            "++ check_python_site_location_is_writable\n",
            "++ python3 -\n",
            "+ [[ 1 == \\0 ]]\n",
            "+ hash automake\n",
            "+ echo 'Installing automake (A dependency for mecab-ko)'\n",
            "Installing automake (A dependency for mecab-ko)\n",
            "+ install_automake\n",
            "+ '[' Linux == Linux ']'\n",
            "++ grep -Ei 'debian|buntu|mint' /etc/lsb-release /etc/os-release\n",
            "+ '[' '/etc/lsb-release:DISTRIB_ID=Ubuntu\n",
            "/etc/lsb-release:DISTRIB_DESCRIPTION=\"Ubuntu 18.04.5 LTS\"\n",
            "/etc/os-release:NAME=\"Ubuntu\"\n",
            "/etc/os-release:ID=ubuntu\n",
            "/etc/os-release:ID_LIKE=debian\n",
            "/etc/os-release:PRETTY_NAME=\"Ubuntu 18.04.5 LTS\"\n",
            "/etc/os-release:HOME_URL=\"https://www.ubuntu.com/\"\n",
            "/etc/os-release:SUPPORT_URL=\"https://help.ubuntu.com/\"\n",
            "/etc/os-release:BUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\n",
            "/etc/os-release:PRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\n",
            "/etc/os-release:UBUNTU_CODENAME=bionic' ']'\n",
            "+ sudo apt-get update\n",
            "Get:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:6 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [41.5 kB]\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Get:8 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:10 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:13 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Hit:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:16 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,703 kB]\n",
            "Get:17 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [872 kB]\n",
            "Fetched 2,889 kB in 3s (876 kB/s)\n",
            "Reading package lists... Done\n",
            "+ sudo apt-get install -y automake\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  autoconf autotools-dev libsigsegv2 m4\n",
            "Suggested packages:\n",
            "  autoconf-archive gnu-standards autoconf-doc libtool gettext m4-doc\n",
            "The following NEW packages will be installed:\n",
            "  autoconf automake autotools-dev libsigsegv2 m4\n",
            "0 upgraded, 5 newly installed, 0 to remove and 17 not upgraded.\n",
            "Need to get 1,082 kB of archives.\n",
            "After this operation, 3,994 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsigsegv2 amd64 2.12-1 [14.7 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 m4 amd64 1.4.18-1 [197 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 autoconf all 2.69-11 [322 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic/main amd64 autotools-dev all 20180224.1 [39.6 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 automake all 1:1.15.1-3ubuntu2 [509 kB]\n",
            "Fetched 1,082 kB in 1s (754 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 5.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libsigsegv2:amd64.\n",
            "(Reading database ... 145480 files and directories currently installed.)\n",
            "Preparing to unpack .../libsigsegv2_2.12-1_amd64.deb ...\n",
            "Unpacking libsigsegv2:amd64 (2.12-1) ...\n",
            "Selecting previously unselected package m4.\n",
            "Preparing to unpack .../archives/m4_1.4.18-1_amd64.deb ...\n",
            "Unpacking m4 (1.4.18-1) ...\n",
            "Selecting previously unselected package autoconf.\n",
            "Preparing to unpack .../autoconf_2.69-11_all.deb ...\n",
            "Unpacking autoconf (2.69-11) ...\n",
            "Selecting previously unselected package autotools-dev.\n",
            "Preparing to unpack .../autotools-dev_20180224.1_all.deb ...\n",
            "Unpacking autotools-dev (20180224.1) ...\n",
            "Selecting previously unselected package automake.\n",
            "Preparing to unpack .../automake_1%3a1.15.1-3ubuntu2_all.deb ...\n",
            "Unpacking automake (1:1.15.1-3ubuntu2) ...\n",
            "Setting up libsigsegv2:amd64 (2.12-1) ...\n",
            "Setting up m4 (1.4.18-1) ...\n",
            "Setting up autotools-dev (20180224.1) ...\n",
            "Setting up autoconf (2.69-11) ...\n",
            "Setting up automake (1:1.15.1-3ubuntu2) ...\n",
            "update-alternatives: using /usr/bin/automake-1.15 to provide /usr/bin/automake (automake) in auto mode\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.2) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "+ hash mecab\n",
            "+ echo 'Install mecab-ko-dic'\n",
            "Install mecab-ko-dic\n",
            "+ install_mecab_ko\n",
            "+ cd /tmp\n",
            "+ curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
            "100 1381k  100 1381k    0     0  1142k      0  0:00:01  0:00:01 --:--:-- 1720k\n",
            "+ tar zxfv mecab-0.996-ko-0.9.2.tar.gz\n",
            "mecab-0.996-ko-0.9.2/\n",
            "mecab-0.996-ko-0.9.2/example/\n",
            "mecab-0.996-ko-0.9.2/example/example.cpp\n",
            "mecab-0.996-ko-0.9.2/example/example_lattice.cpp\n",
            "mecab-0.996-ko-0.9.2/example/example_lattice.c\n",
            "mecab-0.996-ko-0.9.2/example/example.c\n",
            "mecab-0.996-ko-0.9.2/example/thread_test.cpp\n",
            "mecab-0.996-ko-0.9.2/mecab-config.in\n",
            "mecab-0.996-ko-0.9.2/man/\n",
            "mecab-0.996-ko-0.9.2/man/Makefile.am\n",
            "mecab-0.996-ko-0.9.2/man/mecab.1\n",
            "mecab-0.996-ko-0.9.2/man/Makefile.in\n",
            "mecab-0.996-ko-0.9.2/mecab.iss.in\n",
            "mecab-0.996-ko-0.9.2/config.guess\n",
            "mecab-0.996-ko-0.9.2/README\n",
            "mecab-0.996-ko-0.9.2/COPYING\n",
            "mecab-0.996-ko-0.9.2/CHANGES.md\n",
            "mecab-0.996-ko-0.9.2/README.md\n",
            "mecab-0.996-ko-0.9.2/INSTALL\n",
            "mecab-0.996-ko-0.9.2/config.sub\n",
            "mecab-0.996-ko-0.9.2/configure.in\n",
            "mecab-0.996-ko-0.9.2/swig/\n",
            "mecab-0.996-ko-0.9.2/swig/Makefile\n",
            "mecab-0.996-ko-0.9.2/swig/version.h.in\n",
            "mecab-0.996-ko-0.9.2/swig/version.h\n",
            "mecab-0.996-ko-0.9.2/swig/MeCab.i\n",
            "mecab-0.996-ko-0.9.2/aclocal.m4\n",
            "mecab-0.996-ko-0.9.2/LGPL\n",
            "mecab-0.996-ko-0.9.2/Makefile.am\n",
            "mecab-0.996-ko-0.9.2/configure\n",
            "mecab-0.996-ko-0.9.2/tests/\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/test\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/autolink/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/t9/\n",
            "mecab-0.996-ko-0.9.2/tests/t9/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/t9/ipadic.pl\n",
            "mecab-0.996-ko-0.9.2/tests/t9/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/t9/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/t9/test\n",
            "mecab-0.996-ko-0.9.2/tests/t9/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/t9/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/t9/mkdic.pl\n",
            "mecab-0.996-ko-0.9.2/tests/t9/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/ipa.train\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/ipa.test\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/rewrite.def\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/feature.def\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/cost-train/seed/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/run-eval.sh\n",
            "mecab-0.996-ko-0.9.2/tests/run-cost-train.sh\n",
            "mecab-0.996-ko-0.9.2/tests/Makefile.am\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/test\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/katakana/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/eval/\n",
            "mecab-0.996-ko-0.9.2/tests/eval/answer\n",
            "mecab-0.996-ko-0.9.2/tests/eval/system\n",
            "mecab-0.996-ko-0.9.2/tests/eval/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/test\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/mkdic.pl\n",
            "mecab-0.996-ko-0.9.2/tests/shiin/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/latin/\n",
            "mecab-0.996-ko-0.9.2/tests/latin/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/latin/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/latin/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/latin/test\n",
            "mecab-0.996-ko-0.9.2/tests/latin/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/latin/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/latin/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/test\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/chartype/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/run-dics.sh\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/unk.def\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/dicrc\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/dic.csv\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/test\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/char.def\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/matrix.def\n",
            "mecab-0.996-ko-0.9.2/tests/ngram/test.gld\n",
            "mecab-0.996-ko-0.9.2/tests/Makefile.in\n",
            "mecab-0.996-ko-0.9.2/ltmain.sh\n",
            "mecab-0.996-ko-0.9.2/config.rpath\n",
            "mecab-0.996-ko-0.9.2/config.h.in\n",
            "mecab-0.996-ko-0.9.2/mecabrc.in\n",
            "mecab-0.996-ko-0.9.2/GPL\n",
            "mecab-0.996-ko-0.9.2/Makefile.train\n",
            "mecab-0.996-ko-0.9.2/ChangeLog\n",
            "mecab-0.996-ko-0.9.2/install-sh\n",
            "mecab-0.996-ko-0.9.2/AUTHORS\n",
            "mecab-0.996-ko-0.9.2/doc/\n",
            "mecab-0.996-ko-0.9.2/doc/bindings.html\n",
            "mecab-0.996-ko-0.9.2/doc/posid.html\n",
            "mecab-0.996-ko-0.9.2/doc/unk.html\n",
            "mecab-0.996-ko-0.9.2/doc/learn.html\n",
            "mecab-0.996-ko-0.9.2/doc/format.html\n",
            "mecab-0.996-ko-0.9.2/doc/libmecab.html\n",
            "mecab-0.996-ko-0.9.2/doc/mecab.css\n",
            "mecab-0.996-ko-0.9.2/doc/feature.html\n",
            "mecab-0.996-ko-0.9.2/doc/Makefile.am\n",
            "mecab-0.996-ko-0.9.2/doc/soft.html\n",
            "mecab-0.996-ko-0.9.2/doc/en/\n",
            "mecab-0.996-ko-0.9.2/doc/en/bindings.html\n",
            "mecab-0.996-ko-0.9.2/doc/dic-detail.html\n",
            "mecab-0.996-ko-0.9.2/doc/flow.png\n",
            "mecab-0.996-ko-0.9.2/doc/mecab.html\n",
            "mecab-0.996-ko-0.9.2/doc/index.html\n",
            "mecab-0.996-ko-0.9.2/doc/result.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_a.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/globals_eval.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classMeCab_1_1Tagger-members.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/functions_vars.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/doxygen.css\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_r.gif\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classMeCab_1_1Lattice.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/functions.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classMeCab_1_1Tagger.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/mecab_8h_source.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tabs.css\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/nav_f.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_b.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/globals.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/nav_h.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_h.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classMeCab_1_1Model.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/globals_func.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/closed.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_l.gif\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/structmecab__path__t-members.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/functions_func.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/globals_type.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classMeCab_1_1Lattice-members.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/structmecab__node__t.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/namespacemembers_func.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_s.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/structmecab__dictionary__info__t-members.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/namespacemembers_type.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classMeCab_1_1Model-members.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/structmecab__dictionary__info__t.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/namespaces.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/namespacemembers.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/namespaceMeCab.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/structmecab__path__t.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/files.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/structmecab__node__t-members.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/index.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/annotated.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/globals_defs.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/classes.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/mecab_8h-source.html\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/doxygen.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/tab_b.gif\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/bc_s.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/open.png\n",
            "mecab-0.996-ko-0.9.2/doc/doxygen/mecab_8h.html\n",
            "mecab-0.996-ko-0.9.2/doc/dic.html\n",
            "mecab-0.996-ko-0.9.2/doc/partial.html\n",
            "mecab-0.996-ko-0.9.2/doc/feature.png\n",
            "mecab-0.996-ko-0.9.2/doc/Makefile.in\n",
            "mecab-0.996-ko-0.9.2/missing\n",
            "mecab-0.996-ko-0.9.2/BSD\n",
            "mecab-0.996-ko-0.9.2/NEWS\n",
            "mecab-0.996-ko-0.9.2/mkinstalldirs\n",
            "mecab-0.996-ko-0.9.2/src/\n",
            "mecab-0.996-ko-0.9.2/src/dictionary.h\n",
            "mecab-0.996-ko-0.9.2/src/writer.h\n",
            "mecab-0.996-ko-0.9.2/src/utils.h\n",
            "mecab-0.996-ko-0.9.2/src/string_buffer.cpp\n",
            "mecab-0.996-ko-0.9.2/src/tokenizer.cpp\n",
            "mecab-0.996-ko-0.9.2/src/make.bat\n",
            "mecab-0.996-ko-0.9.2/src/mecab.h\n",
            "mecab-0.996-ko-0.9.2/src/freelist.h\n",
            "mecab-0.996-ko-0.9.2/src/string_buffer.h\n",
            "mecab-0.996-ko-0.9.2/src/learner_tagger.h\n",
            "mecab-0.996-ko-0.9.2/src/dictionary_compiler.cpp\n",
            "mecab-0.996-ko-0.9.2/src/eval.cpp\n",
            "mecab-0.996-ko-0.9.2/src/mecab-system-eval.cpp\n",
            "mecab-0.996-ko-0.9.2/src/darts.h\n",
            "mecab-0.996-ko-0.9.2/src/param.h\n",
            "mecab-0.996-ko-0.9.2/src/char_property.h\n",
            "mecab-0.996-ko-0.9.2/src/learner_node.h\n",
            "mecab-0.996-ko-0.9.2/src/mecab-dict-gen.cpp\n",
            "mecab-0.996-ko-0.9.2/src/mecab-dict-index.cpp\n",
            "mecab-0.996-ko-0.9.2/src/winmain.h\n",
            "mecab-0.996-ko-0.9.2/src/thread.h\n",
            "mecab-0.996-ko-0.9.2/src/context_id.cpp\n",
            "mecab-0.996-ko-0.9.2/src/Makefile.am\n",
            "mecab-0.996-ko-0.9.2/src/connector.h\n",
            "mecab-0.996-ko-0.9.2/src/common.h\n",
            "mecab-0.996-ko-0.9.2/src/dictionary_rewriter.cpp\n",
            "mecab-0.996-ko-0.9.2/src/Makefile.msvc.in\n",
            "mecab-0.996-ko-0.9.2/src/dictionary_rewriter.h\n",
            "mecab-0.996-ko-0.9.2/src/feature_index.h\n",
            "mecab-0.996-ko-0.9.2/src/iconv_utils.cpp\n",
            "mecab-0.996-ko-0.9.2/src/char_property.cpp\n",
            "mecab-0.996-ko-0.9.2/src/mecab-test-gen.cpp\n",
            "mecab-0.996-ko-0.9.2/src/tagger.cpp\n",
            "mecab-0.996-ko-0.9.2/src/mecab-cost-train.cpp\n",
            "mecab-0.996-ko-0.9.2/src/learner.cpp\n",
            "mecab-0.996-ko-0.9.2/src/dictionary.cpp\n",
            "mecab-0.996-ko-0.9.2/src/lbfgs.cpp\n",
            "mecab-0.996-ko-0.9.2/src/ucs.h\n",
            "mecab-0.996-ko-0.9.2/src/writer.cpp\n",
            "mecab-0.996-ko-0.9.2/src/learner_tagger.cpp\n",
            "mecab-0.996-ko-0.9.2/src/lbfgs.h\n",
            "mecab-0.996-ko-0.9.2/src/libmecab.cpp\n",
            "mecab-0.996-ko-0.9.2/src/tokenizer.h\n",
            "mecab-0.996-ko-0.9.2/src/mecab.cpp\n",
            "mecab-0.996-ko-0.9.2/src/utils.cpp\n",
            "mecab-0.996-ko-0.9.2/src/dictionary_generator.cpp\n",
            "mecab-0.996-ko-0.9.2/src/param.cpp\n",
            "mecab-0.996-ko-0.9.2/src/context_id.h\n",
            "mecab-0.996-ko-0.9.2/src/mmap.h\n",
            "mecab-0.996-ko-0.9.2/src/viterbi.h\n",
            "mecab-0.996-ko-0.9.2/src/viterbi.cpp\n",
            "mecab-0.996-ko-0.9.2/src/stream_wrapper.h\n",
            "mecab-0.996-ko-0.9.2/src/feature_index.cpp\n",
            "mecab-0.996-ko-0.9.2/src/nbest_generator.h\n",
            "mecab-0.996-ko-0.9.2/src/ucstable.h\n",
            "mecab-0.996-ko-0.9.2/src/nbest_generator.cpp\n",
            "mecab-0.996-ko-0.9.2/src/iconv_utils.h\n",
            "mecab-0.996-ko-0.9.2/src/connector.cpp\n",
            "mecab-0.996-ko-0.9.2/src/Makefile.in\n",
            "mecab-0.996-ko-0.9.2/src/scoped_ptr.h\n",
            "mecab-0.996-ko-0.9.2/Makefile.in\n",
            "+ cd mecab-0.996-ko-0.9.2\n",
            "+ ./configure\n",
            "checking for a BSD-compatible install... /usr/bin/install -c\n",
            "checking whether build environment is sane... yes\n",
            "checking for a thread-safe mkdir -p... /bin/mkdir -p\n",
            "checking for gawk... no\n",
            "checking for mawk... mawk\n",
            "checking whether make sets $(MAKE)... yes\n",
            "checking for gcc... gcc\n",
            "checking whether the C compiler works... yes\n",
            "checking for C compiler default output file name... a.out\n",
            "checking for suffix of executables... \n",
            "checking whether we are cross compiling... no\n",
            "checking for suffix of object files... o\n",
            "checking whether we are using the GNU C compiler... yes\n",
            "checking whether gcc accepts -g... yes\n",
            "checking for gcc option to accept ISO C89... none needed\n",
            "checking for style of include used by make... GNU\n",
            "checking dependency style of gcc... none\n",
            "checking for g++... g++\n",
            "checking whether we are using the GNU C++ compiler... yes\n",
            "checking whether g++ accepts -g... yes\n",
            "checking dependency style of g++... none\n",
            "checking how to run the C preprocessor... gcc -E\n",
            "checking for grep that handles long lines and -e... /bin/grep\n",
            "checking for egrep... /bin/grep -E\n",
            "checking whether gcc needs -traditional... no\n",
            "checking whether make sets $(MAKE)... (cached) yes\n",
            "checking build system type... x86_64-unknown-linux-gnu\n",
            "checking host system type... x86_64-unknown-linux-gnu\n",
            "checking how to print strings... printf\n",
            "checking for a sed that does not truncate output... /bin/sed\n",
            "checking for fgrep... /bin/grep -F\n",
            "checking for ld used by gcc... /usr/bin/ld\n",
            "checking if the linker (/usr/bin/ld) is GNU ld... yes\n",
            "checking for BSD- or MS-compatible name lister (nm)... /usr/bin/nm -B\n",
            "checking the name lister (/usr/bin/nm -B) interface... BSD nm\n",
            "checking whether ln -s works... yes\n",
            "checking the maximum length of command line arguments... 1572864\n",
            "checking whether the shell understands some XSI constructs... yes\n",
            "checking whether the shell understands \"+=\"... yes\n",
            "checking how to convert x86_64-unknown-linux-gnu file names to x86_64-unknown-linux-gnu format... func_convert_file_noop\n",
            "checking how to convert x86_64-unknown-linux-gnu file names to toolchain format... func_convert_file_noop\n",
            "checking for /usr/bin/ld option to reload object files... -r\n",
            "checking for objdump... objdump\n",
            "checking how to recognize dependent libraries... pass_all\n",
            "checking for dlltool... dlltool\n",
            "checking how to associate runtime and link libraries... printf %s\\n\n",
            "checking for ar... ar\n",
            "checking for archiver @FILE support... @\n",
            "checking for strip... strip\n",
            "checking for ranlib... ranlib\n",
            "checking command to parse /usr/bin/nm -B output from gcc object... ok\n",
            "checking for sysroot... no\n",
            "./configure: line 7378: /usr/bin/file: No such file or directory\n",
            "checking for mt... no\n",
            "checking if : is a manifest tool... no\n",
            "checking for ANSI C header files... yes\n",
            "checking for sys/types.h... yes\n",
            "checking for sys/stat.h... yes\n",
            "checking for stdlib.h... yes\n",
            "checking for string.h... yes\n",
            "checking for memory.h... yes\n",
            "checking for strings.h... yes\n",
            "checking for inttypes.h... yes\n",
            "checking for stdint.h... yes\n",
            "checking for unistd.h... yes\n",
            "checking for dlfcn.h... yes\n",
            "checking for objdir... .libs\n",
            "checking if gcc supports -fno-rtti -fno-exceptions... no\n",
            "checking for gcc option to produce PIC... -fPIC -DPIC\n",
            "checking if gcc PIC flag -fPIC -DPIC works... yes\n",
            "checking if gcc static flag -static works... yes\n",
            "checking if gcc supports -c -o file.o... yes\n",
            "checking if gcc supports -c -o file.o... (cached) yes\n",
            "checking whether the gcc linker (/usr/bin/ld) supports shared libraries... yes\n",
            "checking whether -lc should be explicitly linked in... no\n",
            "checking dynamic linker characteristics... GNU/Linux ld.so\n",
            "checking how to hardcode library paths into programs... immediate\n",
            "checking whether stripping libraries is possible... yes\n",
            "checking if libtool supports shared libraries... yes\n",
            "checking whether to build shared libraries... yes\n",
            "checking whether to build static libraries... yes\n",
            "checking how to run the C++ preprocessor... g++ -E\n",
            "checking for ld used by g++... /usr/bin/ld\n",
            "checking if the linker (/usr/bin/ld) is GNU ld... yes\n",
            "checking whether the g++ linker (/usr/bin/ld) supports shared libraries... yes\n",
            "checking for g++ option to produce PIC... -fPIC -DPIC\n",
            "checking if g++ PIC flag -fPIC -DPIC works... yes\n",
            "checking if g++ static flag -static works... yes\n",
            "checking if g++ supports -c -o file.o... yes\n",
            "checking if g++ supports -c -o file.o... (cached) yes\n",
            "checking whether the g++ linker (/usr/bin/ld) supports shared libraries... yes\n",
            "checking dynamic linker characteristics... (cached) GNU/Linux ld.so\n",
            "checking how to hardcode library paths into programs... immediate\n",
            "checking for library containing strerror... none required\n",
            "checking whether byte ordering is bigendian... no\n",
            "checking for ld used by GCC... /usr/bin/ld\n",
            "checking if the linker (/usr/bin/ld) is GNU ld... yes\n",
            "checking for shared library run path origin... done\n",
            "checking for iconv... yes\n",
            "checking for working iconv... yes\n",
            "checking for iconv declaration... \n",
            "         extern size_t iconv (iconv_t cd, char * *inbuf, size_t *inbytesleft, char * *outbuf, size_t *outbytesleft);\n",
            "checking for ANSI C header files... (cached) yes\n",
            "checking for an ANSI C-conforming const... yes\n",
            "checking whether byte ordering is bigendian... (cached) no\n",
            "checking for string.h... (cached) yes\n",
            "checking for stdlib.h... (cached) yes\n",
            "checking for unistd.h... (cached) yes\n",
            "checking fcntl.h usability... yes\n",
            "checking fcntl.h presence... yes\n",
            "checking for fcntl.h... yes\n",
            "checking for stdint.h... (cached) yes\n",
            "checking for sys/stat.h... (cached) yes\n",
            "checking sys/mman.h usability... yes\n",
            "checking sys/mman.h presence... yes\n",
            "checking for sys/mman.h... yes\n",
            "checking sys/times.h usability... yes\n",
            "checking sys/times.h presence... yes\n",
            "checking for sys/times.h... yes\n",
            "checking for sys/types.h... (cached) yes\n",
            "checking dirent.h usability... yes\n",
            "checking dirent.h presence... yes\n",
            "checking for dirent.h... yes\n",
            "checking ctype.h usability... yes\n",
            "checking ctype.h presence... yes\n",
            "checking for ctype.h... yes\n",
            "checking for sys/types.h... (cached) yes\n",
            "checking io.h usability... no\n",
            "checking io.h presence... no\n",
            "checking for io.h... no\n",
            "checking windows.h usability... no\n",
            "checking windows.h presence... no\n",
            "checking for windows.h... no\n",
            "checking pthread.h usability... yes\n",
            "checking pthread.h presence... yes\n",
            "checking for pthread.h... yes\n",
            "checking for off_t... yes\n",
            "checking for size_t... yes\n",
            "checking size of char... 1\n",
            "checking size of short... 2\n",
            "checking size of int... 4\n",
            "checking size of long... 8\n",
            "checking size of long long... 8\n",
            "checking size of size_t... 8\n",
            "checking for size_t... (cached) yes\n",
            "checking for unsigned long long int... yes\n",
            "checking for stdlib.h... (cached) yes\n",
            "checking for unistd.h... (cached) yes\n",
            "checking for sys/param.h... yes\n",
            "checking for getpagesize... yes\n",
            "checking for working mmap... yes\n",
            "checking for main in -lstdc++... yes\n",
            "checking for pthread_create in -lpthread... yes\n",
            "checking for pthread_join in -lpthread... yes\n",
            "checking for getenv... yes\n",
            "checking for opendir... yes\n",
            "checking whether make is GNU Make... yes\n",
            "checking if g++ supports stl <vector> (required)... yes\n",
            "checking if g++ supports stl <list> (required)... yes\n",
            "checking if g++ supports stl <map> (required)... yes\n",
            "checking if g++ supports stl <set> (required)... yes\n",
            "checking if g++ supports stl <queue> (required)... yes\n",
            "checking if g++ supports stl <functional> (required)... yes\n",
            "checking if g++ supports stl <algorithm> (required)... yes\n",
            "checking if g++ supports stl <string> (required)... yes\n",
            "checking if g++ supports stl <iostream> (required)... yes\n",
            "checking if g++ supports stl <sstream> (required)... yes\n",
            "checking if g++ supports stl <fstream> (required)... yes\n",
            "checking if g++ supports template <class T> (required)... yes\n",
            "checking if g++ supports const_cast<> (required)... yes\n",
            "checking if g++ supports static_cast<> (required)... yes\n",
            "checking if g++ supports reinterpret_cast<> (required)... yes\n",
            "checking if g++ supports namespaces (required) ... yes\n",
            "checking if g++ supports __thread (optional)... yes\n",
            "checking if g++ supports template <class T> (required)... yes\n",
            "checking if g++ supports GCC native atomic operations (optional)... yes\n",
            "checking if g++ supports OSX native atomic operations (optional)... no\n",
            "checking if g++ environment provides all required features... yes\n",
            "configure: creating ./config.status\n",
            "config.status: creating Makefile\n",
            "config.status: creating src/Makefile\n",
            "config.status: creating src/Makefile.msvc\n",
            "config.status: creating man/Makefile\n",
            "config.status: creating doc/Makefile\n",
            "config.status: creating tests/Makefile\n",
            "config.status: creating swig/version.h\n",
            "config.status: creating mecab.iss\n",
            "config.status: creating mecab-config\n",
            "config.status: creating mecabrc\n",
            "config.status: creating config.h\n",
            "config.status: executing depfiles commands\n",
            "config.status: executing libtool commands\n",
            "config.status: executing default commands\n",
            "+ make\n",
            "make  all-recursive\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "Making all in src\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o viterbi.lo viterbi.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c viterbi.cpp  -fPIC -DPIC -o .libs/viterbi.o\n",
            "In file included from \u001b[01m\u001b[Kviterbi.cpp:14:0\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[Kparam.h:30:13:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K'\u001b[01m\u001b[KTarget {anonymous}::lexical_cast(Source) [with Target = std::__cxx11::basic_string<char>; Source = std::__cxx11::basic_string<char>]\u001b[m\u001b[K' defined but not used [\u001b[01;35m\u001b[K-Wunused-function\u001b[m\u001b[K]\n",
            " std::string \u001b[01;35m\u001b[Klexical_cast<std::string, std::string>\u001b[m\u001b[K(std::string arg) {\n",
            "             \u001b[01;35m\u001b[K^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[m\u001b[K\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c viterbi.cpp -o viterbi.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o tagger.lo tagger.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c tagger.cpp  -fPIC -DPIC -o .libs/tagger.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c tagger.cpp -o tagger.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o utils.lo utils.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c utils.cpp  -fPIC -DPIC -o .libs/utils.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c utils.cpp -o utils.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o eval.lo eval.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c eval.cpp  -fPIC -DPIC -o .libs/eval.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c eval.cpp -o eval.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o iconv_utils.lo iconv_utils.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c iconv_utils.cpp  -fPIC -DPIC -o .libs/iconv_utils.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c iconv_utils.cpp -o iconv_utils.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o dictionary_rewriter.lo dictionary_rewriter.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary_rewriter.cpp  -fPIC -DPIC -o .libs/dictionary_rewriter.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary_rewriter.cpp -o dictionary_rewriter.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o dictionary_generator.lo dictionary_generator.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary_generator.cpp  -fPIC -DPIC -o .libs/dictionary_generator.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary_generator.cpp -o dictionary_generator.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o dictionary_compiler.lo dictionary_compiler.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary_compiler.cpp  -fPIC -DPIC -o .libs/dictionary_compiler.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary_compiler.cpp -o dictionary_compiler.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o context_id.lo context_id.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c context_id.cpp  -fPIC -DPIC -o .libs/context_id.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c context_id.cpp -o context_id.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o connector.lo connector.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c connector.cpp  -fPIC -DPIC -o .libs/connector.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c connector.cpp -o connector.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o nbest_generator.lo nbest_generator.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c nbest_generator.cpp  -fPIC -DPIC -o .libs/nbest_generator.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c nbest_generator.cpp -o nbest_generator.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o writer.lo writer.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c writer.cpp  -fPIC -DPIC -o .libs/writer.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c writer.cpp -o writer.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o string_buffer.lo string_buffer.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c string_buffer.cpp  -fPIC -DPIC -o .libs/string_buffer.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c string_buffer.cpp -o string_buffer.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o param.lo param.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c param.cpp  -fPIC -DPIC -o .libs/param.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c param.cpp -o param.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o tokenizer.lo tokenizer.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c tokenizer.cpp  -fPIC -DPIC -o .libs/tokenizer.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c tokenizer.cpp -o tokenizer.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o char_property.lo char_property.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c char_property.cpp  -fPIC -DPIC -o .libs/char_property.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c char_property.cpp -o char_property.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o dictionary.lo dictionary.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary.cpp  -fPIC -DPIC -o .libs/dictionary.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c dictionary.cpp -o dictionary.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o feature_index.lo feature_index.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c feature_index.cpp  -fPIC -DPIC -o .libs/feature_index.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c feature_index.cpp -o feature_index.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o lbfgs.lo lbfgs.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c lbfgs.cpp  -fPIC -DPIC -o .libs/lbfgs.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c lbfgs.cpp -o lbfgs.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o learner_tagger.lo learner_tagger.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c learner_tagger.cpp  -fPIC -DPIC -o .libs/learner_tagger.o\n",
            "\u001b[01m\u001b[Klearner_tagger.cpp:25:7:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K'\u001b[01m\u001b[Kchar* MeCab::{anonymous}::mystrdup(const string&)\u001b[m\u001b[K' defined but not used [\u001b[01;35m\u001b[K-Wunused-function\u001b[m\u001b[K]\n",
            " char *\u001b[01;35m\u001b[Kmystrdup\u001b[m\u001b[K(const std::string &str) {\n",
            "       \u001b[01;35m\u001b[K^~~~~~~~\u001b[m\u001b[K\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c learner_tagger.cpp -o learner_tagger.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o learner.lo learner.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c learner.cpp  -fPIC -DPIC -o .libs/learner.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c learner.cpp -o learner.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=compile g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o libmecab.lo libmecab.cpp\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c libmecab.cpp  -fPIC -DPIC -o .libs/libmecab.o\n",
            "libtool: compile:  g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102 -DMECAB_DEFAULT_RC=\\\"/usr/local/etc/mecabrc\\\" -O3 -Wall -c libmecab.cpp -o libmecab.o >/dev/null 2>&1\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall  -no-undefined -version-info 2:0:0  -o libmecab.la -rpath /usr/local/lib viterbi.lo tagger.lo utils.lo eval.lo iconv_utils.lo dictionary_rewriter.lo dictionary_generator.lo dictionary_compiler.lo context_id.lo connector.lo nbest_generator.lo writer.lo string_buffer.lo param.lo tokenizer.lo char_property.lo dictionary.lo feature_index.lo lbfgs.lo learner_tagger.lo learner.lo libmecab.lo  -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++  -fPIC -DPIC -shared -nostdlib /usr/lib/gcc/x86_64-linux-gnu/7/../../../x86_64-linux-gnu/crti.o /usr/lib/gcc/x86_64-linux-gnu/7/crtbeginS.o  .libs/viterbi.o .libs/tagger.o .libs/utils.o .libs/eval.o .libs/iconv_utils.o .libs/dictionary_rewriter.o .libs/dictionary_generator.o .libs/dictionary_compiler.o .libs/context_id.o .libs/connector.o .libs/nbest_generator.o .libs/writer.o .libs/string_buffer.o .libs/param.o .libs/tokenizer.o .libs/char_property.o .libs/dictionary.o .libs/feature_index.o .libs/lbfgs.o .libs/learner_tagger.o .libs/learner.o .libs/libmecab.o   -lpthread -L/usr/lib/gcc/x86_64-linux-gnu/7 -L/usr/lib/gcc/x86_64-linux-gnu/7/../../../x86_64-linux-gnu -L/usr/lib/gcc/x86_64-linux-gnu/7/../../../../lib -L/lib/x86_64-linux-gnu -L/lib/../lib -L/usr/lib/x86_64-linux-gnu -L/usr/lib/../lib -L/usr/local/cuda/lib64/stubs -L/usr/lib/gcc/x86_64-linux-gnu/7/../../.. -lstdc++ -lm -lc -lgcc_s /usr/lib/gcc/x86_64-linux-gnu/7/crtendS.o /usr/lib/gcc/x86_64-linux-gnu/7/../../../x86_64-linux-gnu/crtn.o  -O3   -Wl,-soname -Wl,libmecab.so.2 -o .libs/libmecab.so.2.0.0\n",
            "libtool: link: (cd \".libs\" && rm -f \"libmecab.so.2\" && ln -s \"libmecab.so.2.0.0\" \"libmecab.so.2\")\n",
            "libtool: link: (cd \".libs\" && rm -f \"libmecab.so\" && ln -s \"libmecab.so.2.0.0\" \"libmecab.so\")\n",
            "libtool: link: ar cru .libs/libmecab.a  viterbi.o tagger.o utils.o eval.o iconv_utils.o dictionary_rewriter.o dictionary_generator.o dictionary_compiler.o context_id.o connector.o nbest_generator.o writer.o string_buffer.o param.o tokenizer.o char_property.o dictionary.o feature_index.o lbfgs.o learner_tagger.o learner.o libmecab.o\n",
            "ar: `u' modifier ignored since `D' is the default (see `U')\n",
            "libtool: link: ranlib .libs/libmecab.a\n",
            "libtool: link: ( cd \".libs\" && rm -f \"libmecab.la\" && ln -s \"../libmecab.la\" \"libmecab.la\" )\n",
            "g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o mecab.o mecab.cpp\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall    -o mecab mecab.o libmecab.la -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++ -O3 -Wall -o .libs/mecab mecab.o  ./.libs/libmecab.so -lpthread -lstdc++\n",
            "g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o mecab-dict-index.o mecab-dict-index.cpp\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall    -o mecab-dict-index mecab-dict-index.o libmecab.la -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++ -O3 -Wall -o .libs/mecab-dict-index mecab-dict-index.o  ./.libs/libmecab.so -lpthread -lstdc++\n",
            "g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o mecab-dict-gen.o mecab-dict-gen.cpp\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall    -o mecab-dict-gen mecab-dict-gen.o libmecab.la -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++ -O3 -Wall -o .libs/mecab-dict-gen mecab-dict-gen.o  ./.libs/libmecab.so -lpthread -lstdc++\n",
            "g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o mecab-cost-train.o mecab-cost-train.cpp\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall    -o mecab-cost-train mecab-cost-train.o libmecab.la -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++ -O3 -Wall -o .libs/mecab-cost-train mecab-cost-train.o  ./.libs/libmecab.so -lpthread -lstdc++\n",
            "g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o mecab-system-eval.o mecab-system-eval.cpp\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall    -o mecab-system-eval mecab-system-eval.o libmecab.la -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++ -O3 -Wall -o .libs/mecab-system-eval mecab-system-eval.o  ./.libs/libmecab.so -lpthread -lstdc++\n",
            "g++ -DHAVE_CONFIG_H -I. -I.. -DDIC_VERSION=102   -DMECAB_DEFAULT_RC=\"\\\"/usr/local/etc/mecabrc\\\"\"    -O3 -Wall  -c -o mecab-test-gen.o mecab-test-gen.cpp\n",
            "/bin/bash ../libtool --tag=CXX   --mode=link g++  -O3 -Wall    -o mecab-test-gen mecab-test-gen.o libmecab.la -lpthread -lpthread  -lstdc++ \n",
            "libtool: link: g++ -O3 -Wall -o .libs/mecab-test-gen mecab-test-gen.o  ./.libs/libmecab.so -lpthread -lstdc++\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "Making all in man\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "make[2]: Nothing to be done for 'all'.\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "Making all in doc\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "make[2]: Nothing to be done for 'all'.\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "Making all in tests\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[2]: Nothing to be done for 'all'.\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "+ make check\n",
            "Making check in src\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "make[1]: Nothing to be done for 'check'.\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "Making check in man\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "make[1]: Nothing to be done for 'check'.\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "Making check in doc\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "make[1]: Nothing to be done for 'check'.\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "Making check in tests\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make  check-TESTS\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 2\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 177\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 178x178\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 2\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 83\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 84x84\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 2\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 450\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 1x1\n",
            "\n",
            "done!\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 2\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 162\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 3x3\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 2\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 4\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 1x1\n",
            "\n",
            "done!\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 11\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 1\n",
            "reading ./matrix.def ... 1x1\n",
            "\n",
            "done!\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./unk.def ... 2\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "./pos-id.def is not found. minimum setting is used\n",
            "reading ./dic.csv ... 1\n",
            "reading ./matrix.def ... 1x1\n",
            "\n",
            "done!\n",
            "PASS: run-dics.sh\n",
            "PASS: run-eval.sh\n",
            "seed/pos-id.def is not found. minimum setting is used\n",
            "reading seed/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "seed/model.def is not found. skipped.\n",
            "seed/pos-id.def is not found. minimum setting is used\n",
            "reading seed/dic.csv ... 4335\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading seed/matrix.def ... 1x1\n",
            "\n",
            "done!\n",
            "reading corpus ...\n",
            "Number of sentences: 34\n",
            "Number of features:  64108\n",
            "eta:                 0.00005\n",
            "freq:                1\n",
            "eval-size:           6\n",
            "unk-eval-size:       4\n",
            "threads:             1\n",
            "charset:             EUC-JP\n",
            "C(sigma^2):          1.00000\n",
            "\n",
            "iter=0 err=1.00000 F=0.35771 target=2406.28355 diff=1.00000\n",
            "iter=1 err=0.97059 F=0.65652 target=1484.25231 diff=0.38318\n",
            "iter=2 err=0.91176 F=0.79331 target=863.32765 diff=0.41834\n",
            "iter=3 err=0.85294 F=0.89213 target=596.72480 diff=0.30881\n",
            "iter=4 err=0.61765 F=0.95467 target=336.30744 diff=0.43641\n",
            "iter=5 err=0.50000 F=0.96702 target=246.53039 diff=0.26695\n",
            "iter=6 err=0.35294 F=0.95472 target=188.93963 diff=0.23361\n",
            "iter=7 err=0.20588 F=0.99106 target=168.62665 diff=0.10751\n",
            "iter=8 err=0.05882 F=0.99777 target=158.64865 diff=0.05917\n",
            "iter=9 err=0.08824 F=0.99665 target=154.14530 diff=0.02839\n",
            "iter=10 err=0.08824 F=0.99665 target=151.94257 diff=0.01429\n",
            "iter=11 err=0.02941 F=0.99888 target=147.20825 diff=0.03116\n",
            "iter=12 err=0.00000 F=1.00000 target=147.34956 diff=0.00096\n",
            "iter=13 err=0.02941 F=0.99888 target=146.32592 diff=0.00695\n",
            "iter=14 err=0.00000 F=1.00000 target=145.77299 diff=0.00378\n",
            "iter=15 err=0.02941 F=0.99888 target=145.24641 diff=0.00361\n",
            "iter=16 err=0.00000 F=1.00000 target=144.96490 diff=0.00194\n",
            "iter=17 err=0.02941 F=0.99888 target=144.90246 diff=0.00043\n",
            "iter=18 err=0.00000 F=1.00000 target=144.75959 diff=0.00099\n",
            "iter=19 err=0.00000 F=1.00000 target=144.71727 diff=0.00029\n",
            "iter=20 err=0.00000 F=1.00000 target=144.66337 diff=0.00037\n",
            "iter=21 err=0.00000 F=1.00000 target=144.61349 diff=0.00034\n",
            "iter=22 err=0.00000 F=1.00000 target=144.62987 diff=0.00011\n",
            "iter=23 err=0.00000 F=1.00000 target=144.60060 diff=0.00020\n",
            "iter=24 err=0.00000 F=1.00000 target=144.59125 diff=0.00006\n",
            "iter=25 err=0.00000 F=1.00000 target=144.58619 diff=0.00004\n",
            "iter=26 err=0.00000 F=1.00000 target=144.58219 diff=0.00003\n",
            "iter=27 err=0.00000 F=1.00000 target=144.58059 diff=0.00001\n",
            "\n",
            "Done! writing model file ... \n",
            "model-ipadic.c1.0.f1.model is not a binary model. reopen it as text mode...\n",
            "reading seed/unk.def ... 40\n",
            "reading seed/dic.csv ... 4335\n",
            "emitting model-ipadic.c1.0.f1.dic/left-id.def/ model-ipadic.c1.0.f1.dic/right-id.def\n",
            "emitting model-ipadic.c1.0.f1.dic/unk.def ... 40\n",
            "emitting model-ipadic.c1.0.f1.dic/dic.csv ... 4335\n",
            "emitting matrix      : 100% |###########################################| \n",
            "copying seed/char.def to model-ipadic.c1.0.f1.dic/char.def\n",
            "copying seed/rewrite.def to model-ipadic.c1.0.f1.dic/rewrite.def\n",
            "copying seed/dicrc to model-ipadic.c1.0.f1.dic/dicrc\n",
            "copying seed/feature.def to model-ipadic.c1.0.f1.dic/feature.def\n",
            "copying model-ipadic.c1.0.f1.model to model-ipadic.c1.0.f1.dic/model.def\n",
            "\n",
            "done!\n",
            "model-ipadic.c1.0.f1.dic/pos-id.def is not found. minimum setting is used\n",
            "reading model-ipadic.c1.0.f1.dic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "model-ipadic.c1.0.f1.dic/pos-id.def is not found. minimum setting is used\n",
            "reading model-ipadic.c1.0.f1.dic/dic.csv ... 4335\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading model-ipadic.c1.0.f1.dic/matrix.def ... 346x346\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "              precision          recall         F\n",
            "LEVEL 0:    12.8959(57/442) 11.8998(57/479) 12.3779\n",
            "LEVEL 1:    12.2172(54/442) 11.2735(54/479) 11.7264\n",
            "LEVEL 2:    11.7647(52/442) 10.8559(52/479) 11.2921\n",
            "LEVEL 4:    11.7647(52/442) 10.8559(52/479) 11.2921\n",
            "PASS: run-cost-train.sh\n",
            "==================\n",
            "All 3 tests passed\n",
            "==================\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "+ sudo make install\n",
            "Making install in src\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "test -z \"/usr/local/lib\" || /bin/mkdir -p \"/usr/local/lib\"\n",
            " /bin/bash ../libtool   --mode=install /usr/bin/install -c   libmecab.la '/usr/local/lib'\n",
            "libtool: install: /usr/bin/install -c .libs/libmecab.so.2.0.0 /usr/local/lib/libmecab.so.2.0.0\n",
            "libtool: install: (cd /usr/local/lib && { ln -s -f libmecab.so.2.0.0 libmecab.so.2 || { rm -f libmecab.so.2 && ln -s libmecab.so.2.0.0 libmecab.so.2; }; })\n",
            "libtool: install: (cd /usr/local/lib && { ln -s -f libmecab.so.2.0.0 libmecab.so || { rm -f libmecab.so && ln -s libmecab.so.2.0.0 libmecab.so; }; })\n",
            "libtool: install: /usr/bin/install -c .libs/libmecab.lai /usr/local/lib/libmecab.la\n",
            "libtool: install: /usr/bin/install -c .libs/libmecab.a /usr/local/lib/libmecab.a\n",
            "libtool: install: chmod 644 /usr/local/lib/libmecab.a\n",
            "libtool: install: ranlib /usr/local/lib/libmecab.a\n",
            "libtool: finish: PATH=\"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:/sbin\" ldconfig -n /usr/local/lib\n",
            "----------------------------------------------------------------------\n",
            "Libraries have been installed in:\n",
            "   /usr/local/lib\n",
            "\n",
            "If you ever happen to want to link against installed libraries\n",
            "in a given directory, LIBDIR, you must either use libtool, and\n",
            "specify the full pathname of the library, or use the `-LLIBDIR'\n",
            "flag during linking and do at least one of the following:\n",
            "   - add LIBDIR to the `LD_LIBRARY_PATH' environment variable\n",
            "     during execution\n",
            "   - add LIBDIR to the `LD_RUN_PATH' environment variable\n",
            "     during linking\n",
            "   - use the `-Wl,-rpath -Wl,LIBDIR' linker flag\n",
            "   - have your system administrator add LIBDIR to `/etc/ld.so.conf'\n",
            "\n",
            "See any operating system documentation about shared libraries for\n",
            "more information, such as the ld(1) and ld.so(8) manual pages.\n",
            "----------------------------------------------------------------------\n",
            "test -z \"/usr/local/bin\" || /bin/mkdir -p \"/usr/local/bin\"\n",
            "  /bin/bash ../libtool   --mode=install /usr/bin/install -c mecab '/usr/local/bin'\n",
            "libtool: install: /usr/bin/install -c .libs/mecab /usr/local/bin/mecab\n",
            "test -z \"/usr/local/libexec/mecab\" || /bin/mkdir -p \"/usr/local/libexec/mecab\"\n",
            "  /bin/bash ../libtool   --mode=install /usr/bin/install -c mecab-dict-index mecab-dict-gen mecab-cost-train mecab-system-eval mecab-test-gen '/usr/local/libexec/mecab'\n",
            "libtool: install: /usr/bin/install -c .libs/mecab-dict-index /usr/local/libexec/mecab/mecab-dict-index\n",
            "libtool: install: /usr/bin/install -c .libs/mecab-dict-gen /usr/local/libexec/mecab/mecab-dict-gen\n",
            "libtool: install: /usr/bin/install -c .libs/mecab-cost-train /usr/local/libexec/mecab/mecab-cost-train\n",
            "libtool: install: /usr/bin/install -c .libs/mecab-system-eval /usr/local/libexec/mecab/mecab-system-eval\n",
            "libtool: install: /usr/bin/install -c .libs/mecab-test-gen /usr/local/libexec/mecab/mecab-test-gen\n",
            "test -z \"/usr/local/include\" || /bin/mkdir -p \"/usr/local/include\"\n",
            " /usr/bin/install -c -m 644 mecab.h '/usr/local/include'\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/src'\n",
            "Making install in man\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "make[2]: Nothing to be done for 'install-exec-am'.\n",
            "test -z \"/usr/local/share/man/man1\" || /bin/mkdir -p \"/usr/local/share/man/man1\"\n",
            " /usr/bin/install -c -m 644 mecab.1 '/usr/local/share/man/man1'\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/man'\n",
            "Making install in doc\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "make[2]: Nothing to be done for 'install-exec-am'.\n",
            "make[2]: Nothing to be done for 'install-data-am'.\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/doc'\n",
            "Making install in tests\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[2]: Nothing to be done for 'install-exec-am'.\n",
            "make[2]: Nothing to be done for 'install-data-am'.\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2/tests'\n",
            "make[1]: Entering directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "make[2]: Entering directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "test -z \"/usr/local/bin\" || /bin/mkdir -p \"/usr/local/bin\"\n",
            " /usr/bin/install -c mecab-config '/usr/local/bin'\n",
            "test -z \"/usr/local/etc\" || /bin/mkdir -p \"/usr/local/etc\"\n",
            " /usr/bin/install -c -m 644 mecabrc '/usr/local/etc'\n",
            "make[2]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "make[1]: Leaving directory '/tmp/mecab-0.996-ko-0.9.2'\n",
            "+ [[ -d /usr/local/lib/mecab/dic/mecab-ko-dic ]]\n",
            "+ echo 'Install mecab-ko-dic'\n",
            "Install mecab-ko-dic\n",
            "+ install_mecab_ko_dic\n",
            "+ echo 'Install mecab-ko-dic'\n",
            "Install mecab-ko-dic\n",
            "+ cd /tmp\n",
            "+ curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
            "100 47.4M  100 47.4M    0     0  22.1M      0  0:00:02  0:00:02 --:--:-- 28.5M\n",
            "+ tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz\n",
            "mecab-ko-dic-2.1.1-20180720/\n",
            "mecab-ko-dic-2.1.1-20180720/configure\n",
            "mecab-ko-dic-2.1.1-20180720/COPYING\n",
            "mecab-ko-dic-2.1.1-20180720/autogen.sh\n",
            "mecab-ko-dic-2.1.1-20180720/Place-station.csv\n",
            "mecab-ko-dic-2.1.1-20180720/NNG.csv\n",
            "mecab-ko-dic-2.1.1-20180720/README\n",
            "mecab-ko-dic-2.1.1-20180720/EF.csv\n",
            "mecab-ko-dic-2.1.1-20180720/MAG.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Preanalysis.csv\n",
            "mecab-ko-dic-2.1.1-20180720/NNB.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Person-actor.csv\n",
            "mecab-ko-dic-2.1.1-20180720/VV.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Makefile.in\n",
            "mecab-ko-dic-2.1.1-20180720/matrix.def\n",
            "mecab-ko-dic-2.1.1-20180720/EC.csv\n",
            "mecab-ko-dic-2.1.1-20180720/NNBC.csv\n",
            "mecab-ko-dic-2.1.1-20180720/clean\n",
            "mecab-ko-dic-2.1.1-20180720/ChangeLog\n",
            "mecab-ko-dic-2.1.1-20180720/J.csv\n",
            "mecab-ko-dic-2.1.1-20180720/.keep\n",
            "mecab-ko-dic-2.1.1-20180720/feature.def\n",
            "mecab-ko-dic-2.1.1-20180720/Foreign.csv\n",
            "mecab-ko-dic-2.1.1-20180720/XPN.csv\n",
            "mecab-ko-dic-2.1.1-20180720/EP.csv\n",
            "mecab-ko-dic-2.1.1-20180720/NR.csv\n",
            "mecab-ko-dic-2.1.1-20180720/left-id.def\n",
            "mecab-ko-dic-2.1.1-20180720/Place.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Symbol.csv\n",
            "mecab-ko-dic-2.1.1-20180720/dicrc\n",
            "mecab-ko-dic-2.1.1-20180720/NP.csv\n",
            "mecab-ko-dic-2.1.1-20180720/ETM.csv\n",
            "mecab-ko-dic-2.1.1-20180720/IC.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Place-address.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Group.csv\n",
            "mecab-ko-dic-2.1.1-20180720/model.def\n",
            "mecab-ko-dic-2.1.1-20180720/XSN.csv\n",
            "mecab-ko-dic-2.1.1-20180720/INSTALL\n",
            "mecab-ko-dic-2.1.1-20180720/rewrite.def\n",
            "mecab-ko-dic-2.1.1-20180720/Inflect.csv\n",
            "mecab-ko-dic-2.1.1-20180720/configure.ac\n",
            "mecab-ko-dic-2.1.1-20180720/NNP.csv\n",
            "mecab-ko-dic-2.1.1-20180720/CoinedWord.csv\n",
            "mecab-ko-dic-2.1.1-20180720/XSV.csv\n",
            "mecab-ko-dic-2.1.1-20180720/pos-id.def\n",
            "mecab-ko-dic-2.1.1-20180720/Makefile.am\n",
            "mecab-ko-dic-2.1.1-20180720/unk.def\n",
            "mecab-ko-dic-2.1.1-20180720/missing\n",
            "mecab-ko-dic-2.1.1-20180720/VCP.csv\n",
            "mecab-ko-dic-2.1.1-20180720/install-sh\n",
            "mecab-ko-dic-2.1.1-20180720/Hanja.csv\n",
            "mecab-ko-dic-2.1.1-20180720/MAJ.csv\n",
            "mecab-ko-dic-2.1.1-20180720/XSA.csv\n",
            "mecab-ko-dic-2.1.1-20180720/Wikipedia.csv\n",
            "mecab-ko-dic-2.1.1-20180720/tools/\n",
            "mecab-ko-dic-2.1.1-20180720/tools/add-userdic.sh\n",
            "mecab-ko-dic-2.1.1-20180720/tools/mecab-bestn.sh\n",
            "mecab-ko-dic-2.1.1-20180720/tools/convert_for_using_store.sh\n",
            "mecab-ko-dic-2.1.1-20180720/user-dic/\n",
            "mecab-ko-dic-2.1.1-20180720/user-dic/nnp.csv\n",
            "mecab-ko-dic-2.1.1-20180720/user-dic/place.csv\n",
            "mecab-ko-dic-2.1.1-20180720/user-dic/person.csv\n",
            "mecab-ko-dic-2.1.1-20180720/user-dic/README.md\n",
            "mecab-ko-dic-2.1.1-20180720/NorthKorea.csv\n",
            "mecab-ko-dic-2.1.1-20180720/VX.csv\n",
            "mecab-ko-dic-2.1.1-20180720/right-id.def\n",
            "mecab-ko-dic-2.1.1-20180720/VA.csv\n",
            "mecab-ko-dic-2.1.1-20180720/char.def\n",
            "mecab-ko-dic-2.1.1-20180720/NEWS\n",
            "mecab-ko-dic-2.1.1-20180720/MM.csv\n",
            "mecab-ko-dic-2.1.1-20180720/ETN.csv\n",
            "mecab-ko-dic-2.1.1-20180720/AUTHORS\n",
            "mecab-ko-dic-2.1.1-20180720/Person.csv\n",
            "mecab-ko-dic-2.1.1-20180720/XR.csv\n",
            "mecab-ko-dic-2.1.1-20180720/VCN.csv\n",
            "+ cd mecab-ko-dic-2.1.1-20180720\n",
            "+ ./autogen.sh\n",
            "Looking in current directory for macros.\n",
            "configure.ac:2: warning: AM_INIT_AUTOMAKE: two- and three-arguments forms are deprecated.  For more info, see:\n",
            "configure.ac:2: http://www.gnu.org/software/automake/manual/automake.html#Modernize-AM_005fINIT_005fAUTOMAKE-invocation\n",
            "+ ./configure\n",
            "checking for a BSD-compatible install... /usr/bin/install -c\n",
            "checking whether build environment is sane... yes\n",
            "/tmp/mecab-ko-dic-2.1.1-20180720/missing: Unknown `--is-lightweight' option\n",
            "Try `/tmp/mecab-ko-dic-2.1.1-20180720/missing --help' for more information\n",
            "configure: WARNING: 'missing' script is too old or missing\n",
            "checking for a thread-safe mkdir -p... /bin/mkdir -p\n",
            "checking for gawk... no\n",
            "checking for mawk... mawk\n",
            "checking whether make sets $(MAKE)... yes\n",
            "checking whether make supports nested variables... yes\n",
            "checking for mecab-config... /usr/local/bin/mecab-config\n",
            "checking that generated files are newer than configure... done\n",
            "configure: creating ./config.status\n",
            "config.status: creating Makefile\n",
            "+ [[ Linux == \\L\\i\\n\\u\\x ]]\n",
            "+ mecab-config --libs-only-L\n",
            "+ sudo tee /etc/ld.so.conf.d/mecab.conf\n",
            "/usr/local/lib\n",
            "+ sudo ldconfig\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "+ make\n",
            "/usr/local/libexec/mecab/mecab-dict-index -d . -o . -f UTF-8 -t UTF-8\n",
            "reading ./unk.def ... 13\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./ETM.csv ... 133\n",
            "reading ./NP.csv ... 342\n",
            "reading ./NR.csv ... 482\n",
            "reading ./VCN.csv ... 7\n",
            "reading ./MAG.csv ... 14242\n",
            "reading ./NNB.csv ... 140\n",
            "reading ./Place-station.csv ... 1145\n",
            "reading ./J.csv ... 416\n",
            "reading ./CoinedWord.csv ... 148\n",
            "reading ./EF.csv ... 1820\n",
            "reading ./NNBC.csv ... 677\n",
            "reading ./Wikipedia.csv ... 36762\n",
            "reading ./EP.csv ... 51\n",
            "reading ./MAJ.csv ... 240\n",
            "reading ./Person-actor.csv ... 99230\n",
            "reading ./Inflect.csv ... 44820\n",
            "reading ./Place.csv ... 30303\n",
            "reading ./VCP.csv ... 9\n",
            "reading ./Preanalysis.csv ... 5\n",
            "reading ./VV.csv ... 7331\n",
            "reading ./Person.csv ... 196459\n",
            "reading ./XSV.csv ... 23\n",
            "reading ./IC.csv ... 1305\n",
            "reading ./Group.csv ... 3176\n",
            "reading ./NNP.csv ... 2371\n",
            "reading ./VA.csv ... 2360\n",
            "reading ./ETN.csv ... 14\n",
            "reading ./EC.csv ... 2547\n",
            "reading ./VX.csv ... 125\n",
            "reading ./Place-address.csv ... 19301\n",
            "reading ./Hanja.csv ... 125750\n",
            "reading ./XPN.csv ... 83\n",
            "reading ./MM.csv ... 453\n",
            "reading ./XSN.csv ... 124\n",
            "reading ./Foreign.csv ... 11690\n",
            "reading ./NorthKorea.csv ... 3\n",
            "reading ./Symbol.csv ... 16\n",
            "reading ./NNG.csv ... 208524\n",
            "reading ./XR.csv ... 3637\n",
            "reading ./XSA.csv ... 19\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 3822x2693\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "echo To enable dictionary, rewrite /usr/local/etc/mecabrc as \\\"dicdir = /usr/local/lib/mecab/dic/mecab-ko-dic\\\"\n",
            "To enable dictionary, rewrite /usr/local/etc/mecabrc as \"dicdir = /usr/local/lib/mecab/dic/mecab-ko-dic\"\n",
            "+ sudo sh -c 'echo \"dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic\" > /usr/local/etc/mecabrc'\n",
            "+ sudo make install\n",
            "make[1]: Entering directory '/tmp/mecab-ko-dic-2.1.1-20180720'\n",
            "make[1]: Nothing to be done for 'install-exec-am'.\n",
            " /bin/mkdir -p '/usr/local/lib/mecab/dic/mecab-ko-dic'\n",
            " /usr/bin/install -c -m 644 model.bin matrix.bin char.bin sys.dic unk.dic left-id.def right-id.def rewrite.def pos-id.def dicrc '/usr/local/lib/mecab/dic/mecab-ko-dic'\n",
            "make[1]: Leaving directory '/tmp/mecab-ko-dic-2.1.1-20180720'\n",
            "++ python3 -c 'import pkgutil; print(1 if pkgutil.find_loader(\"MeCab\") else 0)'\n",
            "+ [[ 0 == \\1 ]]\n",
            "+ echo 'Install mecab-python'\n",
            "Install mecab-python\n",
            "+ install_mecab_python\n",
            "+ pushd /tmp\n",
            "/tmp /tmp/mecab-ko-dic-2.1.1-20180720\n",
            "+ [[ ! -d mecab-python-0.996 ]]\n",
            "+ git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git\n",
            "Cloning into 'mecab-python-0.996'...\n",
            "remote: Counting objects: 17, done.\u001b[K\n",
            "remote: Compressing objects: 100% (16/16), done.\u001b[K\n",
            "remote: Total 17 (delta 3), reused 0 (delta 0)\u001b[K\n",
            "Unpacking objects: 100% (17/17), done.\n",
            "+ popd\n",
            "/tmp/mecab-ko-dic-2.1.1-20180720\n",
            "+ python3 -m pip install /tmp/mecab-python-0.996\n",
            "Processing /tmp/mecab-python-0.996\n",
            "Building wheels for collected packages: mecab-python\n",
            "  Building wheel for mecab-python (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mecab-python: filename=mecab_python-0.996_ko_0.9.2-cp36-cp36m-linux_x86_64.whl size=140571 sha256=f8981c2e9da211ba9ec812aecde623faea2a59667d621a674d7826e6df391a12\n",
            "  Stored in directory: /root/.cache/pip/wheels/99/75/a6/e9e73a1dbd73579383644942ef18a6d17ad728a3052a1147fb\n",
            "Successfully built mecab-python\n",
            "Installing collected packages: mecab-python\n",
            "Successfully installed mecab-python-0.996-ko-0.9.2\n",
            "+ echo Done.\n",
            "Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Piur47l5A1TI"
      },
      "source": [
        "from konlpy.tag import Mecab\r\n",
        "import re\r\n",
        "from konlpy.tag import Okt\r\n",
        "\r\n",
        "def text_preprocessing(text_list):\r\n",
        "    \r\n",
        "    stopwords = []\r\n",
        "    tokenizer = Mecab() #형태소 분석기\r\n",
        "\r\n",
        "    token_list = [] \r\n",
        "    \r\n",
        "    for text in text_list:\r\n",
        "        txt = re.sub(\"[a-zA-Z0-9]\", ' ', text) #영문, 숫자 제거 -> 특수문자는 제거하지 않음\r\n",
        "        txt = re.sub('[가-힣\\s]+기자]','기자', txt) #기자 이름 제거\r\n",
        "        token = tokenizer.morphs(txt) #형태소 분석\r\n",
        "\r\n",
        "        token = [t for t in token] \r\n",
        "        token_list.append(token)\r\n",
        "        \r\n",
        "    return token_list, tokenizer\r\n",
        "\r\n",
        "#형태소 분석기를 따로 저장한 이유는 후에 test 데이터 전처리를 진행할 때 이용해야 되기 때문입니다.\r\n",
        "train['new_article'], okt = text_preprocessing(train['content'])"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9fJgEe3joxSc"
      },
      "source": [
        "# 결측치 제거\r\n",
        "train = train[train[\"new_article\"].apply(lambda x: False if len(x)==0 else True)]"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765
        },
        "id": "cAVTxWTDSk_w",
        "outputId": "66661a6d-d47d-4949-9b38-c0e073f4a9b4"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>n_id</th>\n",
              "      <th>date</th>\n",
              "      <th>title</th>\n",
              "      <th>content</th>\n",
              "      <th>ord</th>\n",
              "      <th>info</th>\n",
              "      <th>content_startswith_[</th>\n",
              "      <th>info1_title</th>\n",
              "      <th>info1_content</th>\n",
              "      <th>new_ord</th>\n",
              "      <th>new_article</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>[이데일리 MARKETPOINT]15:32 현재 코스닥 기관 678억 순매도</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>[[, 이, 데일리, ], :, 현재, 코스닥, 기관, 억, 순매도]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>\"실적기반\" 저가에 매집해야 할 8월 급등유망주 TOP 5 전격공개</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>[\", 실적, 기반, \", 저, 가, 에, 매집, 해야, 할, 월, 급등, 유망주,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>하이스탁론, 선취수수료 없는 월 0.4% 최저금리 상품 출시</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>[하이스, 탁론, ,, 선취, 수수료, 없, 는, 월, ., %, 최저, 금리, 상...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NEWS02580</td>\n",
              "      <td>20200605</td>\n",
              "      <td>[마감]코스닥 기관 678억 순매도</td>\n",
              "      <td>종합 경제정보 미디어 이데일리 - 무단전재 &amp; 재배포 금지</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>[종합, 경제, 정보, 미디어, 이, 데일리, -, 무단, 전재, &amp;, 재, 배포,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NEWS09727</td>\n",
              "      <td>20200626</td>\n",
              "      <td>롯데·공영 등 7개 TV 홈쇼핑들, 동행세일 동참</td>\n",
              "      <td>전국적인 소비 붐 조성에 기여할 예정</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.066667</td>\n",
              "      <td>[전국, 적, 인, 소비, 붐, 조성, 에, 기여, 할, 예정]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        n_id  ...                                        new_article\n",
              "0  NEWS02580  ...             [[, 이, 데일리, ], :, 현재, 코스닥, 기관, 억, 순매도]\n",
              "1  NEWS02580  ...  [\", 실적, 기반, \", 저, 가, 에, 매집, 해야, 할, 월, 급등, 유망주,...\n",
              "2  NEWS02580  ...  [하이스, 탁론, ,, 선취, 수수료, 없, 는, 월, ., %, 최저, 금리, 상...\n",
              "3  NEWS02580  ...  [종합, 경제, 정보, 미디어, 이, 데일리, -, 무단, 전재, &, 재, 배포,...\n",
              "4  NEWS09727  ...                [전국, 적, 인, 소비, 붐, 조성, 에, 기여, 할, 예정]\n",
              "\n",
              "[5 rows x 11 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cyht3PbGT7SR"
      },
      "source": [
        "# Vectorization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5O1nOzRK2AR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "524f7599-e2f6-4258-9368-b8c2fc65b7b3"
      },
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\r\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\r\n",
        "max_len = 20\r\n",
        "\r\n",
        "def text2sequence(train_text, max_len=100):\r\n",
        "    \r\n",
        "    tokenizer = Tokenizer()\r\n",
        "    tokenizer.fit_on_texts(train_text)\r\n",
        "    train_X_seq = tokenizer.texts_to_sequences(train_text)\r\n",
        "    vocab_size = len(tokenizer.word_index) + 1\r\n",
        "    print('vocab_size : ', vocab_size)\r\n",
        "    X_train = pad_sequences(train_X_seq, maxlen = max_len, truncating=\"post\") # 길이를 맞춰줌\r\n",
        "    return X_train, vocab_size, tokenizer\r\n",
        "\r\n",
        "train_y = train['info']\r\n",
        "train_X, vocab_size, vectorizer = text2sequence(train['new_article'], max_len = max_len)\r\n",
        "\r\n",
        "print(train_X.shape, train_y.shape)\r\n",
        "\r\n",
        "# train_X new_article을 vectorizer"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "vocab_size :  33798\n",
            "(118676, 20) (118676,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C8iGEGCizF2Z"
      },
      "source": [
        "# Word2Vec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VD2cSguB8uG6"
      },
      "source": [
        "import gensim\r\n",
        "from gensim.models.keyedvectors import KeyedVectors"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sW-x5RpSVORb"
      },
      "source": [
        "# 직접 만든 word2vec model\r\n",
        "#word2vec_customize = gensim.models.Word2Vec.load(\"embedding_300_10_10_all.model\")\r\n",
        "\r\n",
        "word2vec_customize = gensim.models.Word2Vec.load(\"/content/drive/MyDrive/Colab Notebooks/dacon_news/embedding/embedding_300_10_30_all.model\")\r\n",
        "embedding_size = 300"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z_Q7-70BR5oO"
      },
      "source": [
        "# \"\"\"\r\n",
        "# 구글 word2vec 모델\r\n",
        "\r\n",
        "# word2vec_g = gensim.models.KeyedVectors.load_word2vec_format(path+'embedding/GoogleNews-vectors-negative300.bin.gz', binary = True)\r\n",
        "# embedding_size = 300\r\n",
        "# \"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xd5zy8_5fwvT"
      },
      "source": [
        "# 한국어 word2vec model\r\n",
        "# word2vec2 = gensim.models.Word2Vec.load(path+'/embedding/ko.bin')\r\n",
        "# embedding_size = 200\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EdSnbO4xWykD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "324f8665-3c91-47e9-b441-6d19c140a066"
      },
      "source": [
        "embedding_matrix = np.zeros((vocab_size, embedding_size))\r\n",
        "vocab = vectorizer.word_index\r\n",
        "count = 0\r\n",
        "\r\n",
        "for idx, word in enumerate(vocab):\r\n",
        "    if word in word2vec_customize:\r\n",
        "      embedding_vector = word2vec_customize[word]\r\n",
        "      embedding_matrix[idx] = embedding_vector\r\n",
        "    else: # 임베딩 모델에 없는 것\r\n",
        "      #print(word, \"word2vec에 없는 단어입니다.\")\r\n",
        "      count += 1\r\n",
        "      pass\r\n",
        "\r\n",
        "print('count: ', count)  "
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "count:  25472\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__contains__` (Method will be removed in 4.0.0, use self.wv.__contains__() instead).\n",
            "  \n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPK7Dzv8WWWP"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OoqDYSIMXgSE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc992fcc-b464-4b7a-8f2d-7534c9e4c8f4"
      },
      "source": [
        "#concat\r\n",
        "feature_num = 4\r\n",
        "train_X = np.concatenate([train_X,train[[\"info1_title\",\"info1_content\",\"new_ord\",\"content_startswith_[\"]].values.reshape(-1,feature_num)], axis=1)\r\n",
        "train_X\r\n"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.        , 0.        , 0.        , ..., 0.        , 0.25      ,\n",
              "        1.        ],\n",
              "       [0.        , 0.        , 0.        , ..., 1.        , 0.5       ,\n",
              "        0.        ],\n",
              "       [0.        , 0.        , 0.        , ..., 0.        , 0.75      ,\n",
              "        0.        ],\n",
              "       ...,\n",
              "       [0.        , 0.        , 0.        , ..., 0.        , 0.96153846,\n",
              "        0.        ],\n",
              "       [0.        , 0.        , 0.        , ..., 0.        , 0.98076923,\n",
              "        0.        ],\n",
              "       [0.        , 0.        , 0.        , ..., 0.        , 1.        ,\n",
              "        0.        ]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6E9L7Gp6HcxM"
      },
      "source": [
        "#train test split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m-VCzFT3GK-L"
      },
      "source": [
        "\r\n",
        "# 문장별로 train_test set 분리\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "\r\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(train_X, train_y, random_state = 42, test_size = 0.3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p1wCXGq1aAKX",
        "outputId": "8f79ffa7-b8eb-400c-db4e-cb02673d3350"
      },
      "source": [
        "X_train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(83073, 24)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPG9MRVr24ot"
      },
      "source": [
        "index_seq = (X_train.argmax(axis=1)).argsort(axis=0)  # 최대값 index를 반환하여 array 정렬\r\n",
        "X_train = X_train[index_seq]\r\n",
        "y_train = y_train.iloc[index_seq]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vs20YE0UbxzG",
        "outputId": "6ee93613-67b9-4f84-c3c6-e9222340e939"
      },
      "source": [
        "index_seq"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([33419, 74506, 25139, ..., 48543, 48593, 28209])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "05aa4QSydL9n"
      },
      "source": [
        "# stacking 모델링\r\n",
        " GradientBoost,  AdaBoost, CatBoost, ExtraTree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X4gxrTcHQ1ZI",
        "outputId": "f9d0e9cb-a4d4-4110-a288-ccc17a6c0a31"
      },
      "source": [
        "pip install catboost"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting catboost\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/37/bc4e0ddc30c07a96482abf1de7ed1ca54e59bba2026a33bca6d2ef286e5b/catboost-0.24.4-cp36-none-manylinux1_x86_64.whl (65.7MB)\n",
            "\u001b[K     |████████████████████████████████| 65.8MB 68kB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from catboost) (1.4.1)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from catboost) (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.19.4)\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.1.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from catboost) (1.15.0)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from catboost) (4.4.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.8.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (1.3.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2018.9)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (1.3.3)\n",
            "Installing collected packages: catboost\n",
            "Successfully installed catboost-0.24.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_loR1NcfrEhx"
      },
      "source": [
        "from sklearn.ensemble import GradientBoostingClassifier\r\n",
        "from sklearn.tree import DecisionTreeClassifier\r\n",
        "from sklearn.ensemble import AdaBoostClassifier\r\n",
        "import catboost\r\n",
        "from catboost import CatBoostClassifier\r\n",
        "from sklearn.ensemble import ExtraTreesClassifier\r\n",
        "from lightgbm import LGBMClassifier\r\n",
        "\r\n",
        "from xgboost import XGBClassifier"
      ],
      "execution_count": 375,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Tr66gmcq_-b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3931e4e-880b-4b3f-9e2d-7c482224addb"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\r\n",
        "from sklearn.model_selection import GridSearchCV\r\n",
        "from sklearn.model_selection import StratifiedKFold\r\n",
        "\r\n",
        "from sklearn.externals import joblib\r\n",
        "import pickle"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6sjDdMk-gdd_"
      },
      "source": [
        "\r\n",
        "# def auc(m, train, test): \r\n",
        "#     return (metrics.roc_auc_score(y_train,m.predict_proba(train)[:,1]),\r\n",
        "#                            metrics.roc_auc_score(y_test,m.predict_proba(test)[:,1]))\r\n",
        "    \r\n",
        "    \r\n",
        "# cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\r\n",
        "\r\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VyUadDePfdZc"
      },
      "source": [
        "# 모델 로드\r\n",
        "\r\n",
        "#with open('cb_clf.pkl', 'rb') as f:\r\n",
        "#  data = pickle.load(f)\r\n",
        "\r\n",
        "#data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eGvB-9C1dT-F"
      },
      "source": [
        "### GradientBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r_CaeX4vdZNl",
        "outputId": "54e4fcad-65b4-4810-9384-06be7b59b337"
      },
      "source": [
        "from sklearn.ensemble import GradientBoostingClassifier\r\n",
        "\r\n",
        "gb_clf = GradientBoostingClassifier(random_state = 42) # 기본값: max_depth=3, learning_rate=0.1\r\n",
        "gb_clf.fit(X_train, y_train)\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GradientBoostingClassifier(ccp_alpha=0.0, criterion='friedman_mse', init=None,\n",
              "                           learning_rate=0.1, loss='deviance', max_depth=3,\n",
              "                           max_features=None, max_leaf_nodes=None,\n",
              "                           min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                           min_samples_leaf=1, min_samples_split=2,\n",
              "                           min_weight_fraction_leaf=0.0, n_estimators=100,\n",
              "                           n_iter_no_change=None, presort='deprecated',\n",
              "                           random_state=42, subsample=1.0, tol=0.0001,\n",
              "                           validation_fraction=0.1, verbose=0,\n",
              "                           warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mglvFV_jlSsq",
        "outputId": "3ae0774b-1e4c-4804-b68c-1690855b06a8"
      },
      "source": [
        "gb_pred = gb_clf.predict(X_valid)\r\n",
        "\r\n",
        "gb_acc = accuracy_score(y_valid, gb_pred)\r\n",
        "gb_acc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9686824144032806"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_4luByZchZLG",
        "outputId": "2a4286b2-f20d-4994-a00d-0fab683c39e0"
      },
      "source": [
        "gbm_params = {\r\n",
        "    'n_estimators' : [100, 1000],\r\n",
        "    'learning_rate' : [0.01, 0.1],\r\n",
        "}\r\n",
        "\r\n",
        "\r\n",
        "gb_grid = GridSearchCV(gb_clf, param_grid = gbm_params, cv = 3)\r\n",
        "gb_grid.fit(X_train, y_train)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', gb_grid.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(gb_grid.best_score_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'learning_rate': 0.1, 'n_estimators': 1000}\n",
            "최고 예측 정확도: 0.945277\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XoiYEhHtfoOc",
        "outputId": "20be7cd0-7d85-47b9-e5ac-5e78a7b067cb"
      },
      "source": [
        "joblib.dump(gb_grid,'gb_clf.pkl')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['gb_clf.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LCRzIaCTgRig",
        "outputId": "751e5d9b-9c34-4e9b-90c3-9c36b5a129d5"
      },
      "source": [
        "gb_pred = gb_grid.best_estimator_.predict(X_valid)\r\n",
        "\r\n",
        "gb_acc = accuracy_score(y_valid, gb_pred)\r\n",
        "gb_acc\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9865741650984468"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vBVGXIwZhZBV",
        "outputId": "e0583ff9-e155-4fc5-edf3-449c015e14e1"
      },
      "source": [
        "gb_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 1, ..., 0, 1, 1], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZZJQX2Uwq8iS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "outputId": "edbcba93-e8d2-40a1-aea7-ef7116b6bd93"
      },
      "source": [
        "gb_valid_pred = pd.DataFrame(gb_pred, columns =['GradientBoosting'])\r\n",
        "gb_valid_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>GradientBoosting</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35598</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35599</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35600</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35601</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35602</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35603 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       GradientBoosting\n",
              "0                     0\n",
              "1                     0\n",
              "2                     1\n",
              "3                     0\n",
              "4                     1\n",
              "...                 ...\n",
              "35598                 0\n",
              "35599                 0\n",
              "35600                 0\n",
              "35601                 1\n",
              "35602                 1\n",
              "\n",
              "[35603 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ny16Txqhg9de"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "gb_valid_pred.to_csv(\"gb_valid_pred1.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9XsyleuEe_H0"
      },
      "source": [
        "### AdaBoost"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78wX0F5toAkT"
      },
      "source": [
        "- n_estimators를 늘린다면\r\n",
        "\r\n",
        "생성하는 weak learner의 수는 늘어남\r\n",
        "\r\n",
        "이 여러 학습기들의 decision boundary가 많아지면서 모델이 복잡해짐\r\n",
        "\r\n",
        "- learning_rate을 줄인다면\r\n",
        "\r\n",
        "가중치 갱신의 변동폭이 감소해서, 여러 학습기들의 decision boundary 차이가 줄어듦\r\n",
        "\r\n",
        "- n_estimoatrs와 learning_Rate는 trade-off 관계\r\n",
        "n_estimators(또는 learning_rate)를 늘리고, learning_rate(또는 n_estimators)을 줄인다면 서로 효과가 상쇄됌.\r\n",
        "\r\n",
        "→ 두 파라미터를 잘 조정하는 것이 알고리즘의 핵심."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fWVzHfByVCnA",
        "outputId": "d8755bde-b086-433a-9736-d0c37eb9a65a"
      },
      "source": [
        "DTC = DecisionTreeClassifier(max_depth = 1, random_state=42)\r\n",
        "\r\n",
        "ada_clf = AdaBoostClassifier(base_estimator = DTC, random_state=42) # default: estimators = 50, lr =1\r\n",
        "ada_clf.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "AdaBoostClassifier(algorithm='SAMME.R',\n",
              "                   base_estimator=DecisionTreeClassifier(ccp_alpha=0.0,\n",
              "                                                         class_weight=None,\n",
              "                                                         criterion='gini',\n",
              "                                                         max_depth=1,\n",
              "                                                         max_features=None,\n",
              "                                                         max_leaf_nodes=None,\n",
              "                                                         min_impurity_decrease=0.0,\n",
              "                                                         min_impurity_split=None,\n",
              "                                                         min_samples_leaf=1,\n",
              "                                                         min_samples_split=2,\n",
              "                                                         min_weight_fraction_leaf=0.0,\n",
              "                                                         presort='deprecated',\n",
              "                                                         random_state=42,\n",
              "                                                         splitter='best'),\n",
              "                   learning_rate=1.0, n_estimators=50, random_state=42)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tbBmSRVCfA35",
        "outputId": "d06fa631-9ff5-4a19-c3e4-b12f7e2de3dc"
      },
      "source": [
        "ada_pred = ada_clf.predict(X_valid)\r\n",
        "ada_acc = accuracy_score(y_valid, ada_pred)\r\n",
        "ada_acc\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9563239052888801"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cqt360yqfA_c",
        "outputId": "21e09eb9-28dd-41cb-ec69-62fb0846459c"
      },
      "source": [
        "ada_params = {\r\n",
        "    'n_estimators' : [100,1000],\r\n",
        "    'learning_rate' : [0.01, 0.1, 1],\r\n",
        "}\r\n",
        "\r\n",
        "\r\n",
        "ada_grid = GridSearchCV(ada_clf, param_grid = ada_params, cv = 3)\r\n",
        "ada_grid.fit(X_train, y_train)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', ada_grid.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(ada_grid.best_score_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'learning_rate': 1, 'n_estimators': 1000}\n",
            "최고 예측 정확도: 0.926065\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ew_M5w5rgFuf",
        "outputId": "c8218fec-95f5-454c-da79-d983006388cb"
      },
      "source": [
        "joblib.dump(ada_grid,'ada_grid.pkl')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ada_grid.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F_aPIJEjpLEc",
        "outputId": "c8f279c9-37e2-4019-80ba-56a7860cf856"
      },
      "source": [
        "# predict\r\n",
        "ada_pred = ada_grid.best_estimator_.predict(X_valid)\r\n",
        "\r\n",
        "ada_acc = accuracy_score(y_valid, ada_pred)\r\n",
        "ada_acc\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9763222200376372"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "4sNzgzRupWLI",
        "outputId": "2817285d-36a6-4cc2-87f5-6ff3262eca8d"
      },
      "source": [
        "ada_valid_pred = pd.DataFrame(ada_pred, columns =['AdaBoost'])\r\n",
        "ada_valid_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AdaBoost</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35598</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35599</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35600</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35601</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35602</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35603 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       AdaBoost\n",
              "0             0\n",
              "1             0\n",
              "2             1\n",
              "3             0\n",
              "4             1\n",
              "...         ...\n",
              "35598         0\n",
              "35599         0\n",
              "35600         0\n",
              "35601         1\n",
              "35602         1\n",
              "\n",
              "[35603 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6d_YfPZ2O2LW"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "ada_valid_pred.to_csv(\"ada_valid_pred1.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fh9-A5kki8pL"
      },
      "source": [
        "### catBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pmGK8uHvuQ8g",
        "outputId": "420a545d-c079-4e45-b042-e2c1063b447d"
      },
      "source": [
        "cb_clf = CatBoostClassifier(random_state=42)\r\n",
        "cb_clf.fit(X_train, y_train)\r\n",
        "\r\n",
        "cb_pred = cb_clf.predict(X_valid)\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Learning rate set to 0.068005\n",
            "0:\tlearn: 0.5920023\ttotal: 105ms\tremaining: 1m 44s\n",
            "1:\tlearn: 0.5195088\ttotal: 148ms\tremaining: 1m 13s\n",
            "2:\tlearn: 0.4491934\ttotal: 188ms\tremaining: 1m 2s\n",
            "3:\tlearn: 0.3974978\ttotal: 233ms\tremaining: 58.1s\n",
            "4:\tlearn: 0.3606706\ttotal: 274ms\tremaining: 54.5s\n",
            "5:\tlearn: 0.3327232\ttotal: 315ms\tremaining: 52.2s\n",
            "6:\tlearn: 0.3065909\ttotal: 357ms\tremaining: 50.7s\n",
            "7:\tlearn: 0.2837123\ttotal: 406ms\tremaining: 50.3s\n",
            "8:\tlearn: 0.2685990\ttotal: 451ms\tremaining: 49.7s\n",
            "9:\tlearn: 0.2563493\ttotal: 493ms\tremaining: 48.8s\n",
            "10:\tlearn: 0.2455583\ttotal: 531ms\tremaining: 47.7s\n",
            "11:\tlearn: 0.2322818\ttotal: 572ms\tremaining: 47.1s\n",
            "12:\tlearn: 0.2246313\ttotal: 618ms\tremaining: 46.9s\n",
            "13:\tlearn: 0.2162493\ttotal: 659ms\tremaining: 46.4s\n",
            "14:\tlearn: 0.2088295\ttotal: 712ms\tremaining: 46.7s\n",
            "15:\tlearn: 0.2018063\ttotal: 757ms\tremaining: 46.6s\n",
            "16:\tlearn: 0.1959765\ttotal: 798ms\tremaining: 46.2s\n",
            "17:\tlearn: 0.1897272\ttotal: 846ms\tremaining: 46.1s\n",
            "18:\tlearn: 0.1840163\ttotal: 886ms\tremaining: 45.7s\n",
            "19:\tlearn: 0.1797403\ttotal: 927ms\tremaining: 45.4s\n",
            "20:\tlearn: 0.1751887\ttotal: 970ms\tremaining: 45.2s\n",
            "21:\tlearn: 0.1715800\ttotal: 1.01s\tremaining: 45s\n",
            "22:\tlearn: 0.1659592\ttotal: 1.06s\tremaining: 45.1s\n",
            "23:\tlearn: 0.1618923\ttotal: 1.1s\tremaining: 44.8s\n",
            "24:\tlearn: 0.1583492\ttotal: 1.14s\tremaining: 44.6s\n",
            "25:\tlearn: 0.1558957\ttotal: 1.18s\tremaining: 44.4s\n",
            "26:\tlearn: 0.1517327\ttotal: 1.23s\tremaining: 44.2s\n",
            "27:\tlearn: 0.1488430\ttotal: 1.27s\tremaining: 44.1s\n",
            "28:\tlearn: 0.1455637\ttotal: 1.31s\tremaining: 43.9s\n",
            "29:\tlearn: 0.1435616\ttotal: 1.35s\tremaining: 43.7s\n",
            "30:\tlearn: 0.1411111\ttotal: 1.4s\tremaining: 43.8s\n",
            "31:\tlearn: 0.1383575\ttotal: 1.44s\tremaining: 43.7s\n",
            "32:\tlearn: 0.1357645\ttotal: 1.49s\tremaining: 43.7s\n",
            "33:\tlearn: 0.1338624\ttotal: 1.53s\tremaining: 43.6s\n",
            "34:\tlearn: 0.1319724\ttotal: 1.57s\tremaining: 43.4s\n",
            "35:\tlearn: 0.1300642\ttotal: 1.61s\tremaining: 43.2s\n",
            "36:\tlearn: 0.1281445\ttotal: 1.65s\tremaining: 43.1s\n",
            "37:\tlearn: 0.1253788\ttotal: 1.71s\tremaining: 43.3s\n",
            "38:\tlearn: 0.1236997\ttotal: 1.75s\tremaining: 43.2s\n",
            "39:\tlearn: 0.1223831\ttotal: 1.8s\tremaining: 43.1s\n",
            "40:\tlearn: 0.1210987\ttotal: 1.84s\tremaining: 43.1s\n",
            "41:\tlearn: 0.1186960\ttotal: 1.89s\tremaining: 43.1s\n",
            "42:\tlearn: 0.1172339\ttotal: 1.93s\tremaining: 43s\n",
            "43:\tlearn: 0.1156009\ttotal: 1.98s\tremaining: 43s\n",
            "44:\tlearn: 0.1142975\ttotal: 2.02s\tremaining: 42.9s\n",
            "45:\tlearn: 0.1131153\ttotal: 2.06s\tremaining: 42.8s\n",
            "46:\tlearn: 0.1118471\ttotal: 2.1s\tremaining: 42.7s\n",
            "47:\tlearn: 0.1109469\ttotal: 2.15s\tremaining: 42.6s\n",
            "48:\tlearn: 0.1091268\ttotal: 2.19s\tremaining: 42.5s\n",
            "49:\tlearn: 0.1078608\ttotal: 2.23s\tremaining: 42.5s\n",
            "50:\tlearn: 0.1070159\ttotal: 2.28s\tremaining: 42.4s\n",
            "51:\tlearn: 0.1061594\ttotal: 2.32s\tremaining: 42.3s\n",
            "52:\tlearn: 0.1052779\ttotal: 2.36s\tremaining: 42.2s\n",
            "53:\tlearn: 0.1045387\ttotal: 2.41s\tremaining: 42.2s\n",
            "54:\tlearn: 0.1033148\ttotal: 2.45s\tremaining: 42.1s\n",
            "55:\tlearn: 0.1025583\ttotal: 2.49s\tremaining: 42s\n",
            "56:\tlearn: 0.1016192\ttotal: 2.54s\tremaining: 42s\n",
            "57:\tlearn: 0.1003084\ttotal: 2.59s\tremaining: 42s\n",
            "58:\tlearn: 0.0995534\ttotal: 2.63s\tremaining: 41.9s\n",
            "59:\tlearn: 0.0988017\ttotal: 2.67s\tremaining: 41.8s\n",
            "60:\tlearn: 0.0980812\ttotal: 2.72s\tremaining: 41.9s\n",
            "61:\tlearn: 0.0973934\ttotal: 2.76s\tremaining: 41.8s\n",
            "62:\tlearn: 0.0964727\ttotal: 2.81s\tremaining: 41.8s\n",
            "63:\tlearn: 0.0958696\ttotal: 2.85s\tremaining: 41.6s\n",
            "64:\tlearn: 0.0952429\ttotal: 2.89s\tremaining: 41.5s\n",
            "65:\tlearn: 0.0945926\ttotal: 2.93s\tremaining: 41.5s\n",
            "66:\tlearn: 0.0940172\ttotal: 2.97s\tremaining: 41.4s\n",
            "67:\tlearn: 0.0930956\ttotal: 3.01s\tremaining: 41.3s\n",
            "68:\tlearn: 0.0924944\ttotal: 3.06s\tremaining: 41.2s\n",
            "69:\tlearn: 0.0917374\ttotal: 3.1s\tremaining: 41.1s\n",
            "70:\tlearn: 0.0908778\ttotal: 3.14s\tremaining: 41.1s\n",
            "71:\tlearn: 0.0903262\ttotal: 3.18s\tremaining: 41s\n",
            "72:\tlearn: 0.0897525\ttotal: 3.23s\tremaining: 41.1s\n",
            "73:\tlearn: 0.0891032\ttotal: 3.28s\tremaining: 41s\n",
            "74:\tlearn: 0.0882622\ttotal: 3.32s\tremaining: 40.9s\n",
            "75:\tlearn: 0.0878222\ttotal: 3.36s\tremaining: 40.9s\n",
            "76:\tlearn: 0.0873382\ttotal: 3.4s\tremaining: 40.8s\n",
            "77:\tlearn: 0.0869810\ttotal: 3.45s\tremaining: 40.8s\n",
            "78:\tlearn: 0.0862807\ttotal: 3.49s\tremaining: 40.7s\n",
            "79:\tlearn: 0.0857036\ttotal: 3.53s\tremaining: 40.6s\n",
            "80:\tlearn: 0.0852494\ttotal: 3.57s\tremaining: 40.5s\n",
            "81:\tlearn: 0.0848083\ttotal: 3.62s\tremaining: 40.5s\n",
            "82:\tlearn: 0.0842772\ttotal: 3.67s\tremaining: 40.5s\n",
            "83:\tlearn: 0.0839161\ttotal: 3.72s\tremaining: 40.6s\n",
            "84:\tlearn: 0.0834374\ttotal: 3.76s\tremaining: 40.5s\n",
            "85:\tlearn: 0.0830794\ttotal: 3.8s\tremaining: 40.4s\n",
            "86:\tlearn: 0.0827819\ttotal: 3.84s\tremaining: 40.3s\n",
            "87:\tlearn: 0.0823644\ttotal: 3.89s\tremaining: 40.3s\n",
            "88:\tlearn: 0.0820645\ttotal: 3.93s\tremaining: 40.2s\n",
            "89:\tlearn: 0.0817139\ttotal: 3.97s\tremaining: 40.2s\n",
            "90:\tlearn: 0.0813121\ttotal: 4.02s\tremaining: 40.1s\n",
            "91:\tlearn: 0.0808522\ttotal: 4.06s\tremaining: 40.1s\n",
            "92:\tlearn: 0.0805182\ttotal: 4.11s\tremaining: 40s\n",
            "93:\tlearn: 0.0802451\ttotal: 4.15s\tremaining: 40s\n",
            "94:\tlearn: 0.0799190\ttotal: 4.18s\tremaining: 39.9s\n",
            "95:\tlearn: 0.0792677\ttotal: 4.23s\tremaining: 39.8s\n",
            "96:\tlearn: 0.0789129\ttotal: 4.27s\tremaining: 39.8s\n",
            "97:\tlearn: 0.0785278\ttotal: 4.32s\tremaining: 39.8s\n",
            "98:\tlearn: 0.0778693\ttotal: 4.37s\tremaining: 39.7s\n",
            "99:\tlearn: 0.0775069\ttotal: 4.41s\tremaining: 39.7s\n",
            "100:\tlearn: 0.0773001\ttotal: 4.45s\tremaining: 39.6s\n",
            "101:\tlearn: 0.0769550\ttotal: 4.49s\tremaining: 39.5s\n",
            "102:\tlearn: 0.0766586\ttotal: 4.53s\tremaining: 39.5s\n",
            "103:\tlearn: 0.0764126\ttotal: 4.58s\tremaining: 39.5s\n",
            "104:\tlearn: 0.0761796\ttotal: 4.62s\tremaining: 39.4s\n",
            "105:\tlearn: 0.0758357\ttotal: 4.67s\tremaining: 39.4s\n",
            "106:\tlearn: 0.0752733\ttotal: 4.71s\tremaining: 39.3s\n",
            "107:\tlearn: 0.0750042\ttotal: 4.76s\tremaining: 39.3s\n",
            "108:\tlearn: 0.0747883\ttotal: 4.8s\tremaining: 39.2s\n",
            "109:\tlearn: 0.0744456\ttotal: 4.84s\tremaining: 39.2s\n",
            "110:\tlearn: 0.0741586\ttotal: 4.88s\tremaining: 39.1s\n",
            "111:\tlearn: 0.0738319\ttotal: 4.92s\tremaining: 39s\n",
            "112:\tlearn: 0.0732837\ttotal: 4.97s\tremaining: 39s\n",
            "113:\tlearn: 0.0731110\ttotal: 5.02s\tremaining: 39s\n",
            "114:\tlearn: 0.0727551\ttotal: 5.06s\tremaining: 39s\n",
            "115:\tlearn: 0.0724634\ttotal: 5.1s\tremaining: 38.9s\n",
            "116:\tlearn: 0.0722171\ttotal: 5.14s\tremaining: 38.8s\n",
            "117:\tlearn: 0.0719240\ttotal: 5.19s\tremaining: 38.8s\n",
            "118:\tlearn: 0.0717002\ttotal: 5.23s\tremaining: 38.7s\n",
            "119:\tlearn: 0.0714364\ttotal: 5.27s\tremaining: 38.6s\n",
            "120:\tlearn: 0.0712542\ttotal: 5.31s\tremaining: 38.6s\n",
            "121:\tlearn: 0.0709079\ttotal: 5.35s\tremaining: 38.5s\n",
            "122:\tlearn: 0.0706371\ttotal: 5.4s\tremaining: 38.5s\n",
            "123:\tlearn: 0.0703816\ttotal: 5.44s\tremaining: 38.4s\n",
            "124:\tlearn: 0.0699814\ttotal: 5.48s\tremaining: 38.4s\n",
            "125:\tlearn: 0.0695370\ttotal: 5.53s\tremaining: 38.3s\n",
            "126:\tlearn: 0.0693990\ttotal: 5.57s\tremaining: 38.3s\n",
            "127:\tlearn: 0.0690953\ttotal: 5.61s\tremaining: 38.2s\n",
            "128:\tlearn: 0.0687941\ttotal: 5.66s\tremaining: 38.2s\n",
            "129:\tlearn: 0.0683187\ttotal: 5.7s\tremaining: 38.2s\n",
            "130:\tlearn: 0.0681215\ttotal: 5.75s\tremaining: 38.2s\n",
            "131:\tlearn: 0.0679048\ttotal: 5.79s\tremaining: 38.1s\n",
            "132:\tlearn: 0.0677008\ttotal: 5.84s\tremaining: 38.1s\n",
            "133:\tlearn: 0.0675197\ttotal: 5.88s\tremaining: 38s\n",
            "134:\tlearn: 0.0672256\ttotal: 5.92s\tremaining: 37.9s\n",
            "135:\tlearn: 0.0670791\ttotal: 5.96s\tremaining: 37.9s\n",
            "136:\tlearn: 0.0668770\ttotal: 6s\tremaining: 37.8s\n",
            "137:\tlearn: 0.0667712\ttotal: 6.05s\tremaining: 37.8s\n",
            "138:\tlearn: 0.0663066\ttotal: 6.09s\tremaining: 37.7s\n",
            "139:\tlearn: 0.0661280\ttotal: 6.13s\tremaining: 37.7s\n",
            "140:\tlearn: 0.0659282\ttotal: 6.17s\tremaining: 37.6s\n",
            "141:\tlearn: 0.0656503\ttotal: 6.22s\tremaining: 37.6s\n",
            "142:\tlearn: 0.0654561\ttotal: 6.26s\tremaining: 37.5s\n",
            "143:\tlearn: 0.0652903\ttotal: 6.3s\tremaining: 37.5s\n",
            "144:\tlearn: 0.0651229\ttotal: 6.35s\tremaining: 37.4s\n",
            "145:\tlearn: 0.0648412\ttotal: 6.39s\tremaining: 37.4s\n",
            "146:\tlearn: 0.0646475\ttotal: 6.43s\tremaining: 37.3s\n",
            "147:\tlearn: 0.0644821\ttotal: 6.47s\tremaining: 37.3s\n",
            "148:\tlearn: 0.0642504\ttotal: 6.52s\tremaining: 37.2s\n",
            "149:\tlearn: 0.0640714\ttotal: 6.56s\tremaining: 37.2s\n",
            "150:\tlearn: 0.0638540\ttotal: 6.6s\tremaining: 37.1s\n",
            "151:\tlearn: 0.0637240\ttotal: 6.64s\tremaining: 37s\n",
            "152:\tlearn: 0.0636066\ttotal: 6.68s\tremaining: 37s\n",
            "153:\tlearn: 0.0634073\ttotal: 6.73s\tremaining: 37s\n",
            "154:\tlearn: 0.0631846\ttotal: 6.78s\tremaining: 36.9s\n",
            "155:\tlearn: 0.0630323\ttotal: 6.82s\tremaining: 36.9s\n",
            "156:\tlearn: 0.0628877\ttotal: 6.86s\tremaining: 36.8s\n",
            "157:\tlearn: 0.0625920\ttotal: 6.9s\tremaining: 36.8s\n",
            "158:\tlearn: 0.0624421\ttotal: 6.94s\tremaining: 36.7s\n",
            "159:\tlearn: 0.0622716\ttotal: 6.98s\tremaining: 36.7s\n",
            "160:\tlearn: 0.0621613\ttotal: 7.02s\tremaining: 36.6s\n",
            "161:\tlearn: 0.0620168\ttotal: 7.06s\tremaining: 36.5s\n",
            "162:\tlearn: 0.0619409\ttotal: 7.1s\tremaining: 36.5s\n",
            "163:\tlearn: 0.0616592\ttotal: 7.14s\tremaining: 36.4s\n",
            "164:\tlearn: 0.0615227\ttotal: 7.19s\tremaining: 36.4s\n",
            "165:\tlearn: 0.0614020\ttotal: 7.23s\tremaining: 36.3s\n",
            "166:\tlearn: 0.0612342\ttotal: 7.27s\tremaining: 36.3s\n",
            "167:\tlearn: 0.0610762\ttotal: 7.31s\tremaining: 36.2s\n",
            "168:\tlearn: 0.0608811\ttotal: 7.36s\tremaining: 36.2s\n",
            "169:\tlearn: 0.0607811\ttotal: 7.39s\tremaining: 36.1s\n",
            "170:\tlearn: 0.0606686\ttotal: 7.43s\tremaining: 36s\n",
            "171:\tlearn: 0.0604807\ttotal: 7.47s\tremaining: 36s\n",
            "172:\tlearn: 0.0603518\ttotal: 7.52s\tremaining: 36s\n",
            "173:\tlearn: 0.0601971\ttotal: 7.56s\tremaining: 35.9s\n",
            "174:\tlearn: 0.0600340\ttotal: 7.61s\tremaining: 35.9s\n",
            "175:\tlearn: 0.0597591\ttotal: 7.65s\tremaining: 35.8s\n",
            "176:\tlearn: 0.0596253\ttotal: 7.69s\tremaining: 35.8s\n",
            "177:\tlearn: 0.0595077\ttotal: 7.75s\tremaining: 35.8s\n",
            "178:\tlearn: 0.0592341\ttotal: 7.79s\tremaining: 35.7s\n",
            "179:\tlearn: 0.0590200\ttotal: 7.83s\tremaining: 35.7s\n",
            "180:\tlearn: 0.0588749\ttotal: 7.87s\tremaining: 35.6s\n",
            "181:\tlearn: 0.0587861\ttotal: 7.91s\tremaining: 35.6s\n",
            "182:\tlearn: 0.0586174\ttotal: 7.96s\tremaining: 35.5s\n",
            "183:\tlearn: 0.0585239\ttotal: 8.01s\tremaining: 35.5s\n",
            "184:\tlearn: 0.0583612\ttotal: 8.05s\tremaining: 35.5s\n",
            "185:\tlearn: 0.0582415\ttotal: 8.09s\tremaining: 35.4s\n",
            "186:\tlearn: 0.0581429\ttotal: 8.13s\tremaining: 35.3s\n",
            "187:\tlearn: 0.0580629\ttotal: 8.17s\tremaining: 35.3s\n",
            "188:\tlearn: 0.0578931\ttotal: 8.22s\tremaining: 35.3s\n",
            "189:\tlearn: 0.0575436\ttotal: 8.26s\tremaining: 35.2s\n",
            "190:\tlearn: 0.0572870\ttotal: 8.31s\tremaining: 35.2s\n",
            "191:\tlearn: 0.0570043\ttotal: 8.35s\tremaining: 35.2s\n",
            "192:\tlearn: 0.0569017\ttotal: 8.4s\tremaining: 35.1s\n",
            "193:\tlearn: 0.0567599\ttotal: 8.44s\tremaining: 35.1s\n",
            "194:\tlearn: 0.0566363\ttotal: 8.48s\tremaining: 35s\n",
            "195:\tlearn: 0.0565011\ttotal: 8.52s\tremaining: 34.9s\n",
            "196:\tlearn: 0.0563180\ttotal: 8.56s\tremaining: 34.9s\n",
            "197:\tlearn: 0.0561428\ttotal: 8.6s\tremaining: 34.8s\n",
            "198:\tlearn: 0.0560694\ttotal: 8.65s\tremaining: 34.8s\n",
            "199:\tlearn: 0.0559433\ttotal: 8.69s\tremaining: 34.8s\n",
            "200:\tlearn: 0.0557317\ttotal: 8.73s\tremaining: 34.7s\n",
            "201:\tlearn: 0.0555913\ttotal: 8.78s\tremaining: 34.7s\n",
            "202:\tlearn: 0.0554541\ttotal: 8.82s\tremaining: 34.6s\n",
            "203:\tlearn: 0.0552926\ttotal: 8.86s\tremaining: 34.6s\n",
            "204:\tlearn: 0.0552011\ttotal: 8.9s\tremaining: 34.5s\n",
            "205:\tlearn: 0.0550604\ttotal: 8.95s\tremaining: 34.5s\n",
            "206:\tlearn: 0.0549761\ttotal: 8.98s\tremaining: 34.4s\n",
            "207:\tlearn: 0.0548352\ttotal: 9.03s\tremaining: 34.4s\n",
            "208:\tlearn: 0.0546966\ttotal: 9.07s\tremaining: 34.3s\n",
            "209:\tlearn: 0.0545473\ttotal: 9.12s\tremaining: 34.3s\n",
            "210:\tlearn: 0.0544686\ttotal: 9.18s\tremaining: 34.3s\n",
            "211:\tlearn: 0.0543585\ttotal: 9.23s\tremaining: 34.3s\n",
            "212:\tlearn: 0.0542593\ttotal: 9.27s\tremaining: 34.3s\n",
            "213:\tlearn: 0.0539504\ttotal: 9.31s\tremaining: 34.2s\n",
            "214:\tlearn: 0.0537798\ttotal: 9.35s\tremaining: 34.2s\n",
            "215:\tlearn: 0.0536785\ttotal: 9.4s\tremaining: 34.1s\n",
            "216:\tlearn: 0.0536142\ttotal: 9.44s\tremaining: 34s\n",
            "217:\tlearn: 0.0534729\ttotal: 9.49s\tremaining: 34s\n",
            "218:\tlearn: 0.0533722\ttotal: 9.53s\tremaining: 34s\n",
            "219:\tlearn: 0.0532658\ttotal: 9.57s\tremaining: 33.9s\n",
            "220:\tlearn: 0.0531789\ttotal: 9.61s\tremaining: 33.9s\n",
            "221:\tlearn: 0.0531074\ttotal: 9.65s\tremaining: 33.8s\n",
            "222:\tlearn: 0.0529658\ttotal: 9.69s\tremaining: 33.8s\n",
            "223:\tlearn: 0.0527899\ttotal: 9.74s\tremaining: 33.7s\n",
            "224:\tlearn: 0.0527176\ttotal: 9.78s\tremaining: 33.7s\n",
            "225:\tlearn: 0.0525319\ttotal: 9.82s\tremaining: 33.6s\n",
            "226:\tlearn: 0.0523813\ttotal: 9.87s\tremaining: 33.6s\n",
            "227:\tlearn: 0.0522590\ttotal: 9.92s\tremaining: 33.6s\n",
            "228:\tlearn: 0.0521817\ttotal: 9.96s\tremaining: 33.5s\n",
            "229:\tlearn: 0.0520658\ttotal: 10s\tremaining: 33.5s\n",
            "230:\tlearn: 0.0519696\ttotal: 10s\tremaining: 33.4s\n",
            "231:\tlearn: 0.0518675\ttotal: 10.1s\tremaining: 33.4s\n",
            "232:\tlearn: 0.0517651\ttotal: 10.1s\tremaining: 33.4s\n",
            "233:\tlearn: 0.0517240\ttotal: 10.2s\tremaining: 33.3s\n",
            "234:\tlearn: 0.0515527\ttotal: 10.2s\tremaining: 33.2s\n",
            "235:\tlearn: 0.0515022\ttotal: 10.3s\tremaining: 33.2s\n",
            "236:\tlearn: 0.0513875\ttotal: 10.3s\tremaining: 33.1s\n",
            "237:\tlearn: 0.0512748\ttotal: 10.3s\tremaining: 33.1s\n",
            "238:\tlearn: 0.0512177\ttotal: 10.4s\tremaining: 33s\n",
            "239:\tlearn: 0.0510740\ttotal: 10.4s\tremaining: 33s\n",
            "240:\tlearn: 0.0510074\ttotal: 10.5s\tremaining: 32.9s\n",
            "241:\tlearn: 0.0507659\ttotal: 10.5s\tremaining: 32.9s\n",
            "242:\tlearn: 0.0506361\ttotal: 10.6s\tremaining: 32.9s\n",
            "243:\tlearn: 0.0505343\ttotal: 10.6s\tremaining: 32.8s\n",
            "244:\tlearn: 0.0504443\ttotal: 10.6s\tremaining: 32.8s\n",
            "245:\tlearn: 0.0503865\ttotal: 10.7s\tremaining: 32.7s\n",
            "246:\tlearn: 0.0502991\ttotal: 10.7s\tremaining: 32.7s\n",
            "247:\tlearn: 0.0502222\ttotal: 10.8s\tremaining: 32.7s\n",
            "248:\tlearn: 0.0501043\ttotal: 10.8s\tremaining: 32.6s\n",
            "249:\tlearn: 0.0499720\ttotal: 10.9s\tremaining: 32.6s\n",
            "250:\tlearn: 0.0499082\ttotal: 10.9s\tremaining: 32.5s\n",
            "251:\tlearn: 0.0498535\ttotal: 10.9s\tremaining: 32.5s\n",
            "252:\tlearn: 0.0497662\ttotal: 11s\tremaining: 32.4s\n",
            "253:\tlearn: 0.0497000\ttotal: 11s\tremaining: 32.4s\n",
            "254:\tlearn: 0.0494997\ttotal: 11.1s\tremaining: 32.3s\n",
            "255:\tlearn: 0.0493971\ttotal: 11.1s\tremaining: 32.3s\n",
            "256:\tlearn: 0.0493243\ttotal: 11.1s\tremaining: 32.2s\n",
            "257:\tlearn: 0.0492246\ttotal: 11.2s\tremaining: 32.2s\n",
            "258:\tlearn: 0.0491192\ttotal: 11.2s\tremaining: 32.1s\n",
            "259:\tlearn: 0.0490027\ttotal: 11.3s\tremaining: 32.1s\n",
            "260:\tlearn: 0.0489546\ttotal: 11.3s\tremaining: 32s\n",
            "261:\tlearn: 0.0487692\ttotal: 11.4s\tremaining: 32s\n",
            "262:\tlearn: 0.0486747\ttotal: 11.4s\tremaining: 32s\n",
            "263:\tlearn: 0.0485663\ttotal: 11.5s\tremaining: 31.9s\n",
            "264:\tlearn: 0.0484659\ttotal: 11.5s\tremaining: 31.9s\n",
            "265:\tlearn: 0.0484212\ttotal: 11.5s\tremaining: 31.8s\n",
            "266:\tlearn: 0.0483578\ttotal: 11.6s\tremaining: 31.8s\n",
            "267:\tlearn: 0.0482523\ttotal: 11.6s\tremaining: 31.8s\n",
            "268:\tlearn: 0.0481569\ttotal: 11.7s\tremaining: 31.7s\n",
            "269:\tlearn: 0.0480916\ttotal: 11.7s\tremaining: 31.7s\n",
            "270:\tlearn: 0.0480069\ttotal: 11.8s\tremaining: 31.6s\n",
            "271:\tlearn: 0.0479342\ttotal: 11.8s\tremaining: 31.6s\n",
            "272:\tlearn: 0.0477890\ttotal: 11.9s\tremaining: 31.6s\n",
            "273:\tlearn: 0.0477181\ttotal: 11.9s\tremaining: 31.5s\n",
            "274:\tlearn: 0.0476853\ttotal: 11.9s\tremaining: 31.5s\n",
            "275:\tlearn: 0.0475970\ttotal: 12s\tremaining: 31.4s\n",
            "276:\tlearn: 0.0475434\ttotal: 12s\tremaining: 31.4s\n",
            "277:\tlearn: 0.0474295\ttotal: 12.1s\tremaining: 31.3s\n",
            "278:\tlearn: 0.0472862\ttotal: 12.1s\tremaining: 31.3s\n",
            "279:\tlearn: 0.0472093\ttotal: 12.1s\tremaining: 31.2s\n",
            "280:\tlearn: 0.0471257\ttotal: 12.2s\tremaining: 31.2s\n",
            "281:\tlearn: 0.0470344\ttotal: 12.2s\tremaining: 31.1s\n",
            "282:\tlearn: 0.0469570\ttotal: 12.3s\tremaining: 31.1s\n",
            "283:\tlearn: 0.0468823\ttotal: 12.3s\tremaining: 31.1s\n",
            "284:\tlearn: 0.0468267\ttotal: 12.4s\tremaining: 31s\n",
            "285:\tlearn: 0.0467143\ttotal: 12.4s\tremaining: 31s\n",
            "286:\tlearn: 0.0466050\ttotal: 12.4s\tremaining: 30.9s\n",
            "287:\tlearn: 0.0465639\ttotal: 12.5s\tremaining: 30.9s\n",
            "288:\tlearn: 0.0464731\ttotal: 12.5s\tremaining: 30.8s\n",
            "289:\tlearn: 0.0464246\ttotal: 12.6s\tremaining: 30.8s\n",
            "290:\tlearn: 0.0463071\ttotal: 12.6s\tremaining: 30.7s\n",
            "291:\tlearn: 0.0462766\ttotal: 12.7s\tremaining: 30.7s\n",
            "292:\tlearn: 0.0462030\ttotal: 12.7s\tremaining: 30.6s\n",
            "293:\tlearn: 0.0461472\ttotal: 12.7s\tremaining: 30.6s\n",
            "294:\tlearn: 0.0460557\ttotal: 12.8s\tremaining: 30.6s\n",
            "295:\tlearn: 0.0459406\ttotal: 12.8s\tremaining: 30.5s\n",
            "296:\tlearn: 0.0459014\ttotal: 12.9s\tremaining: 30.5s\n",
            "297:\tlearn: 0.0457587\ttotal: 12.9s\tremaining: 30.4s\n",
            "298:\tlearn: 0.0456648\ttotal: 13s\tremaining: 30.4s\n",
            "299:\tlearn: 0.0455818\ttotal: 13s\tremaining: 30.3s\n",
            "300:\tlearn: 0.0454942\ttotal: 13s\tremaining: 30.3s\n",
            "301:\tlearn: 0.0454601\ttotal: 13.1s\tremaining: 30.2s\n",
            "302:\tlearn: 0.0453780\ttotal: 13.1s\tremaining: 30.2s\n",
            "303:\tlearn: 0.0453177\ttotal: 13.2s\tremaining: 30.1s\n",
            "304:\tlearn: 0.0452737\ttotal: 13.2s\tremaining: 30.1s\n",
            "305:\tlearn: 0.0451822\ttotal: 13.3s\tremaining: 30.1s\n",
            "306:\tlearn: 0.0451280\ttotal: 13.3s\tremaining: 30s\n",
            "307:\tlearn: 0.0450559\ttotal: 13.3s\tremaining: 30s\n",
            "308:\tlearn: 0.0450037\ttotal: 13.4s\tremaining: 29.9s\n",
            "309:\tlearn: 0.0449235\ttotal: 13.4s\tremaining: 29.9s\n",
            "310:\tlearn: 0.0448867\ttotal: 13.5s\tremaining: 29.8s\n",
            "311:\tlearn: 0.0448392\ttotal: 13.5s\tremaining: 29.8s\n",
            "312:\tlearn: 0.0447808\ttotal: 13.5s\tremaining: 29.7s\n",
            "313:\tlearn: 0.0446804\ttotal: 13.6s\tremaining: 29.7s\n",
            "314:\tlearn: 0.0446303\ttotal: 13.6s\tremaining: 29.6s\n",
            "315:\tlearn: 0.0445638\ttotal: 13.7s\tremaining: 29.6s\n",
            "316:\tlearn: 0.0444936\ttotal: 13.7s\tremaining: 29.5s\n",
            "317:\tlearn: 0.0444440\ttotal: 13.8s\tremaining: 29.5s\n",
            "318:\tlearn: 0.0443397\ttotal: 13.8s\tremaining: 29.5s\n",
            "319:\tlearn: 0.0442543\ttotal: 13.9s\tremaining: 29.4s\n",
            "320:\tlearn: 0.0441747\ttotal: 13.9s\tremaining: 29.4s\n",
            "321:\tlearn: 0.0440964\ttotal: 13.9s\tremaining: 29.3s\n",
            "322:\tlearn: 0.0440438\ttotal: 14s\tremaining: 29.3s\n",
            "323:\tlearn: 0.0439392\ttotal: 14s\tremaining: 29.2s\n",
            "324:\tlearn: 0.0438625\ttotal: 14.1s\tremaining: 29.2s\n",
            "325:\tlearn: 0.0438063\ttotal: 14.1s\tremaining: 29.2s\n",
            "326:\tlearn: 0.0437221\ttotal: 14.1s\tremaining: 29.1s\n",
            "327:\tlearn: 0.0436360\ttotal: 14.2s\tremaining: 29.1s\n",
            "328:\tlearn: 0.0435584\ttotal: 14.2s\tremaining: 29s\n",
            "329:\tlearn: 0.0434598\ttotal: 14.3s\tremaining: 29s\n",
            "330:\tlearn: 0.0433666\ttotal: 14.3s\tremaining: 28.9s\n",
            "331:\tlearn: 0.0432447\ttotal: 14.4s\tremaining: 28.9s\n",
            "332:\tlearn: 0.0431378\ttotal: 14.4s\tremaining: 28.9s\n",
            "333:\tlearn: 0.0430817\ttotal: 14.5s\tremaining: 28.8s\n",
            "334:\tlearn: 0.0430281\ttotal: 14.5s\tremaining: 28.8s\n",
            "335:\tlearn: 0.0429423\ttotal: 14.5s\tremaining: 28.7s\n",
            "336:\tlearn: 0.0428960\ttotal: 14.6s\tremaining: 28.7s\n",
            "337:\tlearn: 0.0428612\ttotal: 14.6s\tremaining: 28.6s\n",
            "338:\tlearn: 0.0427707\ttotal: 14.7s\tremaining: 28.6s\n",
            "339:\tlearn: 0.0427197\ttotal: 14.7s\tremaining: 28.6s\n",
            "340:\tlearn: 0.0426682\ttotal: 14.8s\tremaining: 28.5s\n",
            "341:\tlearn: 0.0426224\ttotal: 14.8s\tremaining: 28.5s\n",
            "342:\tlearn: 0.0425449\ttotal: 14.8s\tremaining: 28.4s\n",
            "343:\tlearn: 0.0424912\ttotal: 14.9s\tremaining: 28.4s\n",
            "344:\tlearn: 0.0424294\ttotal: 14.9s\tremaining: 28.3s\n",
            "345:\tlearn: 0.0424078\ttotal: 15s\tremaining: 28.3s\n",
            "346:\tlearn: 0.0423617\ttotal: 15s\tremaining: 28.2s\n",
            "347:\tlearn: 0.0422934\ttotal: 15s\tremaining: 28.2s\n",
            "348:\tlearn: 0.0421583\ttotal: 15.1s\tremaining: 28.1s\n",
            "349:\tlearn: 0.0420937\ttotal: 15.1s\tremaining: 28.1s\n",
            "350:\tlearn: 0.0420358\ttotal: 15.2s\tremaining: 28.1s\n",
            "351:\tlearn: 0.0420107\ttotal: 15.2s\tremaining: 28s\n",
            "352:\tlearn: 0.0419422\ttotal: 15.3s\tremaining: 28s\n",
            "353:\tlearn: 0.0419049\ttotal: 15.3s\tremaining: 27.9s\n",
            "354:\tlearn: 0.0418818\ttotal: 15.3s\tremaining: 27.9s\n",
            "355:\tlearn: 0.0417994\ttotal: 15.4s\tremaining: 27.8s\n",
            "356:\tlearn: 0.0416841\ttotal: 15.4s\tremaining: 27.8s\n",
            "357:\tlearn: 0.0416117\ttotal: 15.5s\tremaining: 27.7s\n",
            "358:\tlearn: 0.0415453\ttotal: 15.5s\tremaining: 27.7s\n",
            "359:\tlearn: 0.0415010\ttotal: 15.6s\tremaining: 27.7s\n",
            "360:\tlearn: 0.0414383\ttotal: 15.6s\tremaining: 27.6s\n",
            "361:\tlearn: 0.0413778\ttotal: 15.6s\tremaining: 27.6s\n",
            "362:\tlearn: 0.0413241\ttotal: 15.7s\tremaining: 27.5s\n",
            "363:\tlearn: 0.0412869\ttotal: 15.7s\tremaining: 27.5s\n",
            "364:\tlearn: 0.0412344\ttotal: 15.8s\tremaining: 27.4s\n",
            "365:\tlearn: 0.0411862\ttotal: 15.8s\tremaining: 27.4s\n",
            "366:\tlearn: 0.0410943\ttotal: 15.9s\tremaining: 27.4s\n",
            "367:\tlearn: 0.0410395\ttotal: 15.9s\tremaining: 27.3s\n",
            "368:\tlearn: 0.0409719\ttotal: 15.9s\tremaining: 27.3s\n",
            "369:\tlearn: 0.0409334\ttotal: 16s\tremaining: 27.2s\n",
            "370:\tlearn: 0.0409051\ttotal: 16s\tremaining: 27.2s\n",
            "371:\tlearn: 0.0408467\ttotal: 16.1s\tremaining: 27.1s\n",
            "372:\tlearn: 0.0407864\ttotal: 16.1s\tremaining: 27.1s\n",
            "373:\tlearn: 0.0407473\ttotal: 16.2s\tremaining: 27s\n",
            "374:\tlearn: 0.0406786\ttotal: 16.2s\tremaining: 27s\n",
            "375:\tlearn: 0.0406234\ttotal: 16.2s\tremaining: 27s\n",
            "376:\tlearn: 0.0404927\ttotal: 16.3s\tremaining: 26.9s\n",
            "377:\tlearn: 0.0404046\ttotal: 16.3s\tremaining: 26.9s\n",
            "378:\tlearn: 0.0403199\ttotal: 16.4s\tremaining: 26.8s\n",
            "379:\tlearn: 0.0402922\ttotal: 16.4s\tremaining: 26.8s\n",
            "380:\tlearn: 0.0402495\ttotal: 16.5s\tremaining: 26.7s\n",
            "381:\tlearn: 0.0401994\ttotal: 16.5s\tremaining: 26.7s\n",
            "382:\tlearn: 0.0401741\ttotal: 16.5s\tremaining: 26.6s\n",
            "383:\tlearn: 0.0401093\ttotal: 16.6s\tremaining: 26.6s\n",
            "384:\tlearn: 0.0400247\ttotal: 16.6s\tremaining: 26.6s\n",
            "385:\tlearn: 0.0399807\ttotal: 16.7s\tremaining: 26.5s\n",
            "386:\tlearn: 0.0399338\ttotal: 16.7s\tremaining: 26.5s\n",
            "387:\tlearn: 0.0398662\ttotal: 16.8s\tremaining: 26.4s\n",
            "388:\tlearn: 0.0398364\ttotal: 16.8s\tremaining: 26.4s\n",
            "389:\tlearn: 0.0397559\ttotal: 16.8s\tremaining: 26.4s\n",
            "390:\tlearn: 0.0397177\ttotal: 16.9s\tremaining: 26.3s\n",
            "391:\tlearn: 0.0396725\ttotal: 16.9s\tremaining: 26.3s\n",
            "392:\tlearn: 0.0396216\ttotal: 17s\tremaining: 26.2s\n",
            "393:\tlearn: 0.0395592\ttotal: 17s\tremaining: 26.2s\n",
            "394:\tlearn: 0.0394871\ttotal: 17.1s\tremaining: 26.1s\n",
            "395:\tlearn: 0.0394136\ttotal: 17.1s\tremaining: 26.1s\n",
            "396:\tlearn: 0.0393370\ttotal: 17.1s\tremaining: 26s\n",
            "397:\tlearn: 0.0393064\ttotal: 17.2s\tremaining: 26s\n",
            "398:\tlearn: 0.0392691\ttotal: 17.2s\tremaining: 25.9s\n",
            "399:\tlearn: 0.0392468\ttotal: 17.3s\tremaining: 25.9s\n",
            "400:\tlearn: 0.0392183\ttotal: 17.3s\tremaining: 25.8s\n",
            "401:\tlearn: 0.0391772\ttotal: 17.3s\tremaining: 25.8s\n",
            "402:\tlearn: 0.0390845\ttotal: 17.4s\tremaining: 25.8s\n",
            "403:\tlearn: 0.0390198\ttotal: 17.4s\tremaining: 25.7s\n",
            "404:\tlearn: 0.0389657\ttotal: 17.5s\tremaining: 25.7s\n",
            "405:\tlearn: 0.0389382\ttotal: 17.5s\tremaining: 25.6s\n",
            "406:\tlearn: 0.0388676\ttotal: 17.6s\tremaining: 25.6s\n",
            "407:\tlearn: 0.0388000\ttotal: 17.6s\tremaining: 25.5s\n",
            "408:\tlearn: 0.0387381\ttotal: 17.6s\tremaining: 25.5s\n",
            "409:\tlearn: 0.0386606\ttotal: 17.7s\tremaining: 25.4s\n",
            "410:\tlearn: 0.0385902\ttotal: 17.7s\tremaining: 25.4s\n",
            "411:\tlearn: 0.0385451\ttotal: 17.8s\tremaining: 25.4s\n",
            "412:\tlearn: 0.0385096\ttotal: 17.8s\tremaining: 25.3s\n",
            "413:\tlearn: 0.0384620\ttotal: 17.9s\tremaining: 25.3s\n",
            "414:\tlearn: 0.0384274\ttotal: 17.9s\tremaining: 25.2s\n",
            "415:\tlearn: 0.0383847\ttotal: 18s\tremaining: 25.2s\n",
            "416:\tlearn: 0.0383459\ttotal: 18s\tremaining: 25.2s\n",
            "417:\tlearn: 0.0382924\ttotal: 18s\tremaining: 25.1s\n",
            "418:\tlearn: 0.0382395\ttotal: 18.1s\tremaining: 25.1s\n",
            "419:\tlearn: 0.0381994\ttotal: 18.1s\tremaining: 25s\n",
            "420:\tlearn: 0.0381183\ttotal: 18.2s\tremaining: 25s\n",
            "421:\tlearn: 0.0380607\ttotal: 18.2s\tremaining: 24.9s\n",
            "422:\tlearn: 0.0379584\ttotal: 18.3s\tremaining: 24.9s\n",
            "423:\tlearn: 0.0379108\ttotal: 18.3s\tremaining: 24.9s\n",
            "424:\tlearn: 0.0378645\ttotal: 18.4s\tremaining: 24.8s\n",
            "425:\tlearn: 0.0377704\ttotal: 18.4s\tremaining: 24.8s\n",
            "426:\tlearn: 0.0377275\ttotal: 18.5s\tremaining: 24.8s\n",
            "427:\tlearn: 0.0376887\ttotal: 18.5s\tremaining: 24.7s\n",
            "428:\tlearn: 0.0376380\ttotal: 18.5s\tremaining: 24.7s\n",
            "429:\tlearn: 0.0375880\ttotal: 18.6s\tremaining: 24.6s\n",
            "430:\tlearn: 0.0374938\ttotal: 18.6s\tremaining: 24.6s\n",
            "431:\tlearn: 0.0374660\ttotal: 18.7s\tremaining: 24.6s\n",
            "432:\tlearn: 0.0373742\ttotal: 18.7s\tremaining: 24.5s\n",
            "433:\tlearn: 0.0373377\ttotal: 18.8s\tremaining: 24.5s\n",
            "434:\tlearn: 0.0373206\ttotal: 18.8s\tremaining: 24.4s\n",
            "435:\tlearn: 0.0372724\ttotal: 18.9s\tremaining: 24.4s\n",
            "436:\tlearn: 0.0372329\ttotal: 18.9s\tremaining: 24.3s\n",
            "437:\tlearn: 0.0371993\ttotal: 18.9s\tremaining: 24.3s\n",
            "438:\tlearn: 0.0371432\ttotal: 19s\tremaining: 24.3s\n",
            "439:\tlearn: 0.0370599\ttotal: 19s\tremaining: 24.2s\n",
            "440:\tlearn: 0.0370276\ttotal: 19.1s\tremaining: 24.2s\n",
            "441:\tlearn: 0.0369830\ttotal: 19.1s\tremaining: 24.1s\n",
            "442:\tlearn: 0.0368888\ttotal: 19.2s\tremaining: 24.1s\n",
            "443:\tlearn: 0.0368488\ttotal: 19.2s\tremaining: 24.1s\n",
            "444:\tlearn: 0.0367944\ttotal: 19.3s\tremaining: 24s\n",
            "445:\tlearn: 0.0367529\ttotal: 19.3s\tremaining: 24s\n",
            "446:\tlearn: 0.0366871\ttotal: 19.3s\tremaining: 23.9s\n",
            "447:\tlearn: 0.0366242\ttotal: 19.4s\tremaining: 23.9s\n",
            "448:\tlearn: 0.0365708\ttotal: 19.4s\tremaining: 23.8s\n",
            "449:\tlearn: 0.0365248\ttotal: 19.5s\tremaining: 23.8s\n",
            "450:\tlearn: 0.0364867\ttotal: 19.5s\tremaining: 23.8s\n",
            "451:\tlearn: 0.0364480\ttotal: 19.6s\tremaining: 23.7s\n",
            "452:\tlearn: 0.0364133\ttotal: 19.6s\tremaining: 23.7s\n",
            "453:\tlearn: 0.0363703\ttotal: 19.6s\tremaining: 23.6s\n",
            "454:\tlearn: 0.0363030\ttotal: 19.7s\tremaining: 23.6s\n",
            "455:\tlearn: 0.0362781\ttotal: 19.8s\tremaining: 23.6s\n",
            "456:\tlearn: 0.0362071\ttotal: 19.8s\tremaining: 23.5s\n",
            "457:\tlearn: 0.0361590\ttotal: 19.8s\tremaining: 23.5s\n",
            "458:\tlearn: 0.0361283\ttotal: 19.9s\tremaining: 23.4s\n",
            "459:\tlearn: 0.0361141\ttotal: 19.9s\tremaining: 23.4s\n",
            "460:\tlearn: 0.0360834\ttotal: 20s\tremaining: 23.3s\n",
            "461:\tlearn: 0.0360533\ttotal: 20s\tremaining: 23.3s\n",
            "462:\tlearn: 0.0360158\ttotal: 20.1s\tremaining: 23.3s\n",
            "463:\tlearn: 0.0359814\ttotal: 20.1s\tremaining: 23.2s\n",
            "464:\tlearn: 0.0359561\ttotal: 20.1s\tremaining: 23.2s\n",
            "465:\tlearn: 0.0358971\ttotal: 20.2s\tremaining: 23.1s\n",
            "466:\tlearn: 0.0358136\ttotal: 20.2s\tremaining: 23.1s\n",
            "467:\tlearn: 0.0357492\ttotal: 20.3s\tremaining: 23s\n",
            "468:\tlearn: 0.0357041\ttotal: 20.3s\tremaining: 23s\n",
            "469:\tlearn: 0.0356587\ttotal: 20.3s\tremaining: 22.9s\n",
            "470:\tlearn: 0.0356219\ttotal: 20.4s\tremaining: 22.9s\n",
            "471:\tlearn: 0.0356086\ttotal: 20.4s\tremaining: 22.8s\n",
            "472:\tlearn: 0.0355635\ttotal: 20.5s\tremaining: 22.8s\n",
            "473:\tlearn: 0.0355109\ttotal: 20.5s\tremaining: 22.7s\n",
            "474:\tlearn: 0.0354708\ttotal: 20.5s\tremaining: 22.7s\n",
            "475:\tlearn: 0.0354617\ttotal: 20.6s\tremaining: 22.7s\n",
            "476:\tlearn: 0.0354479\ttotal: 20.6s\tremaining: 22.6s\n",
            "477:\tlearn: 0.0354084\ttotal: 20.7s\tremaining: 22.6s\n",
            "478:\tlearn: 0.0353695\ttotal: 20.7s\tremaining: 22.5s\n",
            "479:\tlearn: 0.0353539\ttotal: 20.7s\tremaining: 22.5s\n",
            "480:\tlearn: 0.0353282\ttotal: 20.8s\tremaining: 22.4s\n",
            "481:\tlearn: 0.0353107\ttotal: 20.8s\tremaining: 22.4s\n",
            "482:\tlearn: 0.0352587\ttotal: 20.9s\tremaining: 22.3s\n",
            "483:\tlearn: 0.0352094\ttotal: 20.9s\tremaining: 22.3s\n",
            "484:\tlearn: 0.0351575\ttotal: 21s\tremaining: 22.3s\n",
            "485:\tlearn: 0.0350882\ttotal: 21s\tremaining: 22.2s\n",
            "486:\tlearn: 0.0350608\ttotal: 21s\tremaining: 22.2s\n",
            "487:\tlearn: 0.0350310\ttotal: 21.1s\tremaining: 22.1s\n",
            "488:\tlearn: 0.0349958\ttotal: 21.1s\tremaining: 22.1s\n",
            "489:\tlearn: 0.0349626\ttotal: 21.2s\tremaining: 22s\n",
            "490:\tlearn: 0.0349253\ttotal: 21.2s\tremaining: 22s\n",
            "491:\tlearn: 0.0348587\ttotal: 21.2s\tremaining: 21.9s\n",
            "492:\tlearn: 0.0348111\ttotal: 21.3s\tremaining: 21.9s\n",
            "493:\tlearn: 0.0347851\ttotal: 21.3s\tremaining: 21.8s\n",
            "494:\tlearn: 0.0347322\ttotal: 21.4s\tremaining: 21.8s\n",
            "495:\tlearn: 0.0347004\ttotal: 21.4s\tremaining: 21.8s\n",
            "496:\tlearn: 0.0346610\ttotal: 21.5s\tremaining: 21.7s\n",
            "497:\tlearn: 0.0346241\ttotal: 21.5s\tremaining: 21.7s\n",
            "498:\tlearn: 0.0345908\ttotal: 21.5s\tremaining: 21.6s\n",
            "499:\tlearn: 0.0345378\ttotal: 21.6s\tremaining: 21.6s\n",
            "500:\tlearn: 0.0344985\ttotal: 21.6s\tremaining: 21.5s\n",
            "501:\tlearn: 0.0344672\ttotal: 21.7s\tremaining: 21.5s\n",
            "502:\tlearn: 0.0344558\ttotal: 21.7s\tremaining: 21.5s\n",
            "503:\tlearn: 0.0344224\ttotal: 21.7s\tremaining: 21.4s\n",
            "504:\tlearn: 0.0343834\ttotal: 21.8s\tremaining: 21.4s\n",
            "505:\tlearn: 0.0343518\ttotal: 21.8s\tremaining: 21.3s\n",
            "506:\tlearn: 0.0343116\ttotal: 21.9s\tremaining: 21.3s\n",
            "507:\tlearn: 0.0342681\ttotal: 21.9s\tremaining: 21.2s\n",
            "508:\tlearn: 0.0342200\ttotal: 22s\tremaining: 21.2s\n",
            "509:\tlearn: 0.0341820\ttotal: 22s\tremaining: 21.1s\n",
            "510:\tlearn: 0.0341233\ttotal: 22s\tremaining: 21.1s\n",
            "511:\tlearn: 0.0341054\ttotal: 22.1s\tremaining: 21.1s\n",
            "512:\tlearn: 0.0340551\ttotal: 22.1s\tremaining: 21s\n",
            "513:\tlearn: 0.0340194\ttotal: 22.2s\tremaining: 21s\n",
            "514:\tlearn: 0.0340037\ttotal: 22.2s\tremaining: 20.9s\n",
            "515:\tlearn: 0.0339683\ttotal: 22.3s\tremaining: 20.9s\n",
            "516:\tlearn: 0.0339246\ttotal: 22.3s\tremaining: 20.8s\n",
            "517:\tlearn: 0.0339082\ttotal: 22.3s\tremaining: 20.8s\n",
            "518:\tlearn: 0.0338418\ttotal: 22.4s\tremaining: 20.8s\n",
            "519:\tlearn: 0.0337935\ttotal: 22.4s\tremaining: 20.7s\n",
            "520:\tlearn: 0.0337354\ttotal: 22.5s\tremaining: 20.7s\n",
            "521:\tlearn: 0.0336601\ttotal: 22.5s\tremaining: 20.6s\n",
            "522:\tlearn: 0.0335880\ttotal: 22.6s\tremaining: 20.6s\n",
            "523:\tlearn: 0.0335537\ttotal: 22.6s\tremaining: 20.5s\n",
            "524:\tlearn: 0.0335016\ttotal: 22.7s\tremaining: 20.5s\n",
            "525:\tlearn: 0.0334835\ttotal: 22.7s\tremaining: 20.4s\n",
            "526:\tlearn: 0.0334534\ttotal: 22.7s\tremaining: 20.4s\n",
            "527:\tlearn: 0.0333972\ttotal: 22.8s\tremaining: 20.4s\n",
            "528:\tlearn: 0.0333480\ttotal: 22.8s\tremaining: 20.3s\n",
            "529:\tlearn: 0.0333102\ttotal: 22.9s\tremaining: 20.3s\n",
            "530:\tlearn: 0.0332742\ttotal: 22.9s\tremaining: 20.2s\n",
            "531:\tlearn: 0.0332435\ttotal: 22.9s\tremaining: 20.2s\n",
            "532:\tlearn: 0.0332232\ttotal: 23s\tremaining: 20.1s\n",
            "533:\tlearn: 0.0331675\ttotal: 23s\tremaining: 20.1s\n",
            "534:\tlearn: 0.0331352\ttotal: 23.1s\tremaining: 20s\n",
            "535:\tlearn: 0.0331221\ttotal: 23.1s\tremaining: 20s\n",
            "536:\tlearn: 0.0330983\ttotal: 23.1s\tremaining: 19.9s\n",
            "537:\tlearn: 0.0330433\ttotal: 23.2s\tremaining: 19.9s\n",
            "538:\tlearn: 0.0330274\ttotal: 23.2s\tremaining: 19.9s\n",
            "539:\tlearn: 0.0330090\ttotal: 23.3s\tremaining: 19.8s\n",
            "540:\tlearn: 0.0329704\ttotal: 23.3s\tremaining: 19.8s\n",
            "541:\tlearn: 0.0329391\ttotal: 23.3s\tremaining: 19.7s\n",
            "542:\tlearn: 0.0329077\ttotal: 23.4s\tremaining: 19.7s\n",
            "543:\tlearn: 0.0328551\ttotal: 23.4s\tremaining: 19.6s\n",
            "544:\tlearn: 0.0328081\ttotal: 23.5s\tremaining: 19.6s\n",
            "545:\tlearn: 0.0327899\ttotal: 23.5s\tremaining: 19.6s\n",
            "546:\tlearn: 0.0327431\ttotal: 23.6s\tremaining: 19.5s\n",
            "547:\tlearn: 0.0327046\ttotal: 23.6s\tremaining: 19.5s\n",
            "548:\tlearn: 0.0326506\ttotal: 23.7s\tremaining: 19.4s\n",
            "549:\tlearn: 0.0326052\ttotal: 23.7s\tremaining: 19.4s\n",
            "550:\tlearn: 0.0325646\ttotal: 23.7s\tremaining: 19.3s\n",
            "551:\tlearn: 0.0325277\ttotal: 23.8s\tremaining: 19.3s\n",
            "552:\tlearn: 0.0324853\ttotal: 23.8s\tremaining: 19.3s\n",
            "553:\tlearn: 0.0324532\ttotal: 23.9s\tremaining: 19.2s\n",
            "554:\tlearn: 0.0324229\ttotal: 23.9s\tremaining: 19.2s\n",
            "555:\tlearn: 0.0323932\ttotal: 24s\tremaining: 19.1s\n",
            "556:\tlearn: 0.0323673\ttotal: 24s\tremaining: 19.1s\n",
            "557:\tlearn: 0.0323039\ttotal: 24s\tremaining: 19s\n",
            "558:\tlearn: 0.0322600\ttotal: 24.1s\tremaining: 19s\n",
            "559:\tlearn: 0.0322274\ttotal: 24.1s\tremaining: 19s\n",
            "560:\tlearn: 0.0321940\ttotal: 24.2s\tremaining: 18.9s\n",
            "561:\tlearn: 0.0321505\ttotal: 24.2s\tremaining: 18.9s\n",
            "562:\tlearn: 0.0321029\ttotal: 24.3s\tremaining: 18.8s\n",
            "563:\tlearn: 0.0320886\ttotal: 24.3s\tremaining: 18.8s\n",
            "564:\tlearn: 0.0320549\ttotal: 24.3s\tremaining: 18.7s\n",
            "565:\tlearn: 0.0320237\ttotal: 24.4s\tremaining: 18.7s\n",
            "566:\tlearn: 0.0320039\ttotal: 24.4s\tremaining: 18.6s\n",
            "567:\tlearn: 0.0319597\ttotal: 24.5s\tremaining: 18.6s\n",
            "568:\tlearn: 0.0319461\ttotal: 24.5s\tremaining: 18.6s\n",
            "569:\tlearn: 0.0319064\ttotal: 24.6s\tremaining: 18.5s\n",
            "570:\tlearn: 0.0318792\ttotal: 24.6s\tremaining: 18.5s\n",
            "571:\tlearn: 0.0318540\ttotal: 24.6s\tremaining: 18.4s\n",
            "572:\tlearn: 0.0318094\ttotal: 24.7s\tremaining: 18.4s\n",
            "573:\tlearn: 0.0317797\ttotal: 24.7s\tremaining: 18.3s\n",
            "574:\tlearn: 0.0317272\ttotal: 24.8s\tremaining: 18.3s\n",
            "575:\tlearn: 0.0317002\ttotal: 24.8s\tremaining: 18.3s\n",
            "576:\tlearn: 0.0316791\ttotal: 24.8s\tremaining: 18.2s\n",
            "577:\tlearn: 0.0316331\ttotal: 24.9s\tremaining: 18.2s\n",
            "578:\tlearn: 0.0315970\ttotal: 24.9s\tremaining: 18.1s\n",
            "579:\tlearn: 0.0315570\ttotal: 25s\tremaining: 18.1s\n",
            "580:\tlearn: 0.0315112\ttotal: 25s\tremaining: 18s\n",
            "581:\tlearn: 0.0314882\ttotal: 25.1s\tremaining: 18s\n",
            "582:\tlearn: 0.0314512\ttotal: 25.1s\tremaining: 17.9s\n",
            "583:\tlearn: 0.0313683\ttotal: 25.1s\tremaining: 17.9s\n",
            "584:\tlearn: 0.0313193\ttotal: 25.2s\tremaining: 17.9s\n",
            "585:\tlearn: 0.0312878\ttotal: 25.2s\tremaining: 17.8s\n",
            "586:\tlearn: 0.0312639\ttotal: 25.3s\tremaining: 17.8s\n",
            "587:\tlearn: 0.0312230\ttotal: 25.3s\tremaining: 17.7s\n",
            "588:\tlearn: 0.0311739\ttotal: 25.4s\tremaining: 17.7s\n",
            "589:\tlearn: 0.0311371\ttotal: 25.4s\tremaining: 17.6s\n",
            "590:\tlearn: 0.0310775\ttotal: 25.4s\tremaining: 17.6s\n",
            "591:\tlearn: 0.0310612\ttotal: 25.5s\tremaining: 17.6s\n",
            "592:\tlearn: 0.0310380\ttotal: 25.5s\tremaining: 17.5s\n",
            "593:\tlearn: 0.0310045\ttotal: 25.6s\tremaining: 17.5s\n",
            "594:\tlearn: 0.0309777\ttotal: 25.6s\tremaining: 17.4s\n",
            "595:\tlearn: 0.0309437\ttotal: 25.7s\tremaining: 17.4s\n",
            "596:\tlearn: 0.0309117\ttotal: 25.7s\tremaining: 17.3s\n",
            "597:\tlearn: 0.0308657\ttotal: 25.7s\tremaining: 17.3s\n",
            "598:\tlearn: 0.0308189\ttotal: 25.8s\tremaining: 17.3s\n",
            "599:\tlearn: 0.0308091\ttotal: 25.8s\tremaining: 17.2s\n",
            "600:\tlearn: 0.0307710\ttotal: 25.9s\tremaining: 17.2s\n",
            "601:\tlearn: 0.0307154\ttotal: 25.9s\tremaining: 17.1s\n",
            "602:\tlearn: 0.0306751\ttotal: 26s\tremaining: 17.1s\n",
            "603:\tlearn: 0.0306692\ttotal: 26s\tremaining: 17s\n",
            "604:\tlearn: 0.0306302\ttotal: 26s\tremaining: 17s\n",
            "605:\tlearn: 0.0306042\ttotal: 26.1s\tremaining: 17s\n",
            "606:\tlearn: 0.0305799\ttotal: 26.1s\tremaining: 16.9s\n",
            "607:\tlearn: 0.0305746\ttotal: 26.2s\tremaining: 16.9s\n",
            "608:\tlearn: 0.0305296\ttotal: 26.2s\tremaining: 16.8s\n",
            "609:\tlearn: 0.0304840\ttotal: 26.2s\tremaining: 16.8s\n",
            "610:\tlearn: 0.0304513\ttotal: 26.3s\tremaining: 16.7s\n",
            "611:\tlearn: 0.0304125\ttotal: 26.3s\tremaining: 16.7s\n",
            "612:\tlearn: 0.0303898\ttotal: 26.4s\tremaining: 16.6s\n",
            "613:\tlearn: 0.0303623\ttotal: 26.4s\tremaining: 16.6s\n",
            "614:\tlearn: 0.0303160\ttotal: 26.4s\tremaining: 16.6s\n",
            "615:\tlearn: 0.0302562\ttotal: 26.5s\tremaining: 16.5s\n",
            "616:\tlearn: 0.0302333\ttotal: 26.5s\tremaining: 16.5s\n",
            "617:\tlearn: 0.0301964\ttotal: 26.6s\tremaining: 16.4s\n",
            "618:\tlearn: 0.0301500\ttotal: 26.6s\tremaining: 16.4s\n",
            "619:\tlearn: 0.0301287\ttotal: 26.7s\tremaining: 16.3s\n",
            "620:\tlearn: 0.0300916\ttotal: 26.7s\tremaining: 16.3s\n",
            "621:\tlearn: 0.0300508\ttotal: 26.7s\tremaining: 16.3s\n",
            "622:\tlearn: 0.0300072\ttotal: 26.8s\tremaining: 16.2s\n",
            "623:\tlearn: 0.0299907\ttotal: 26.8s\tremaining: 16.2s\n",
            "624:\tlearn: 0.0299557\ttotal: 26.9s\tremaining: 16.1s\n",
            "625:\tlearn: 0.0299130\ttotal: 26.9s\tremaining: 16.1s\n",
            "626:\tlearn: 0.0298824\ttotal: 27s\tremaining: 16s\n",
            "627:\tlearn: 0.0298535\ttotal: 27s\tremaining: 16s\n",
            "628:\tlearn: 0.0298055\ttotal: 27s\tremaining: 16s\n",
            "629:\tlearn: 0.0297659\ttotal: 27.1s\tremaining: 15.9s\n",
            "630:\tlearn: 0.0297321\ttotal: 27.1s\tremaining: 15.9s\n",
            "631:\tlearn: 0.0296949\ttotal: 27.2s\tremaining: 15.8s\n",
            "632:\tlearn: 0.0296555\ttotal: 27.2s\tremaining: 15.8s\n",
            "633:\tlearn: 0.0295982\ttotal: 27.3s\tremaining: 15.7s\n",
            "634:\tlearn: 0.0295765\ttotal: 27.3s\tremaining: 15.7s\n",
            "635:\tlearn: 0.0295442\ttotal: 27.3s\tremaining: 15.6s\n",
            "636:\tlearn: 0.0295038\ttotal: 27.4s\tremaining: 15.6s\n",
            "637:\tlearn: 0.0294676\ttotal: 27.4s\tremaining: 15.6s\n",
            "638:\tlearn: 0.0294049\ttotal: 27.5s\tremaining: 15.5s\n",
            "639:\tlearn: 0.0293774\ttotal: 27.5s\tremaining: 15.5s\n",
            "640:\tlearn: 0.0293477\ttotal: 27.6s\tremaining: 15.4s\n",
            "641:\tlearn: 0.0293187\ttotal: 27.6s\tremaining: 15.4s\n",
            "642:\tlearn: 0.0292680\ttotal: 27.7s\tremaining: 15.4s\n",
            "643:\tlearn: 0.0292404\ttotal: 27.7s\tremaining: 15.3s\n",
            "644:\tlearn: 0.0292191\ttotal: 27.7s\tremaining: 15.3s\n",
            "645:\tlearn: 0.0291848\ttotal: 27.8s\tremaining: 15.2s\n",
            "646:\tlearn: 0.0291228\ttotal: 27.8s\tremaining: 15.2s\n",
            "647:\tlearn: 0.0290975\ttotal: 27.9s\tremaining: 15.1s\n",
            "648:\tlearn: 0.0290731\ttotal: 27.9s\tremaining: 15.1s\n",
            "649:\tlearn: 0.0290376\ttotal: 28s\tremaining: 15.1s\n",
            "650:\tlearn: 0.0289941\ttotal: 28s\tremaining: 15s\n",
            "651:\tlearn: 0.0289754\ttotal: 28.1s\tremaining: 15s\n",
            "652:\tlearn: 0.0289390\ttotal: 28.1s\tremaining: 14.9s\n",
            "653:\tlearn: 0.0289033\ttotal: 28.2s\tremaining: 14.9s\n",
            "654:\tlearn: 0.0288698\ttotal: 28.2s\tremaining: 14.9s\n",
            "655:\tlearn: 0.0288365\ttotal: 28.2s\tremaining: 14.8s\n",
            "656:\tlearn: 0.0288204\ttotal: 28.3s\tremaining: 14.8s\n",
            "657:\tlearn: 0.0287904\ttotal: 28.3s\tremaining: 14.7s\n",
            "658:\tlearn: 0.0287847\ttotal: 28.4s\tremaining: 14.7s\n",
            "659:\tlearn: 0.0287458\ttotal: 28.4s\tremaining: 14.6s\n",
            "660:\tlearn: 0.0287257\ttotal: 28.5s\tremaining: 14.6s\n",
            "661:\tlearn: 0.0287085\ttotal: 28.5s\tremaining: 14.5s\n",
            "662:\tlearn: 0.0286959\ttotal: 28.5s\tremaining: 14.5s\n",
            "663:\tlearn: 0.0286699\ttotal: 28.6s\tremaining: 14.5s\n",
            "664:\tlearn: 0.0286279\ttotal: 28.6s\tremaining: 14.4s\n",
            "665:\tlearn: 0.0285914\ttotal: 28.7s\tremaining: 14.4s\n",
            "666:\tlearn: 0.0285591\ttotal: 28.7s\tremaining: 14.3s\n",
            "667:\tlearn: 0.0285314\ttotal: 28.8s\tremaining: 14.3s\n",
            "668:\tlearn: 0.0285076\ttotal: 28.8s\tremaining: 14.3s\n",
            "669:\tlearn: 0.0285013\ttotal: 28.9s\tremaining: 14.2s\n",
            "670:\tlearn: 0.0284553\ttotal: 28.9s\tremaining: 14.2s\n",
            "671:\tlearn: 0.0284162\ttotal: 29s\tremaining: 14.1s\n",
            "672:\tlearn: 0.0283350\ttotal: 29s\tremaining: 14.1s\n",
            "673:\tlearn: 0.0283110\ttotal: 29s\tremaining: 14s\n",
            "674:\tlearn: 0.0282777\ttotal: 29.1s\tremaining: 14s\n",
            "675:\tlearn: 0.0282626\ttotal: 29.1s\tremaining: 14s\n",
            "676:\tlearn: 0.0282469\ttotal: 29.2s\tremaining: 13.9s\n",
            "677:\tlearn: 0.0282299\ttotal: 29.2s\tremaining: 13.9s\n",
            "678:\tlearn: 0.0281924\ttotal: 29.3s\tremaining: 13.8s\n",
            "679:\tlearn: 0.0281653\ttotal: 29.3s\tremaining: 13.8s\n",
            "680:\tlearn: 0.0281402\ttotal: 29.3s\tremaining: 13.7s\n",
            "681:\tlearn: 0.0280793\ttotal: 29.4s\tremaining: 13.7s\n",
            "682:\tlearn: 0.0280256\ttotal: 29.4s\tremaining: 13.7s\n",
            "683:\tlearn: 0.0279787\ttotal: 29.5s\tremaining: 13.6s\n",
            "684:\tlearn: 0.0279669\ttotal: 29.5s\tremaining: 13.6s\n",
            "685:\tlearn: 0.0279255\ttotal: 29.6s\tremaining: 13.5s\n",
            "686:\tlearn: 0.0278941\ttotal: 29.6s\tremaining: 13.5s\n",
            "687:\tlearn: 0.0278698\ttotal: 29.6s\tremaining: 13.4s\n",
            "688:\tlearn: 0.0278445\ttotal: 29.7s\tremaining: 13.4s\n",
            "689:\tlearn: 0.0278243\ttotal: 29.7s\tremaining: 13.4s\n",
            "690:\tlearn: 0.0277945\ttotal: 29.8s\tremaining: 13.3s\n",
            "691:\tlearn: 0.0277713\ttotal: 29.8s\tremaining: 13.3s\n",
            "692:\tlearn: 0.0277340\ttotal: 29.9s\tremaining: 13.2s\n",
            "693:\tlearn: 0.0277234\ttotal: 29.9s\tremaining: 13.2s\n",
            "694:\tlearn: 0.0276894\ttotal: 29.9s\tremaining: 13.1s\n",
            "695:\tlearn: 0.0276600\ttotal: 30s\tremaining: 13.1s\n",
            "696:\tlearn: 0.0276446\ttotal: 30s\tremaining: 13.1s\n",
            "697:\tlearn: 0.0276301\ttotal: 30.1s\tremaining: 13s\n",
            "698:\tlearn: 0.0276057\ttotal: 30.1s\tremaining: 13s\n",
            "699:\tlearn: 0.0275661\ttotal: 30.2s\tremaining: 12.9s\n",
            "700:\tlearn: 0.0275263\ttotal: 30.2s\tremaining: 12.9s\n",
            "701:\tlearn: 0.0274929\ttotal: 30.2s\tremaining: 12.8s\n",
            "702:\tlearn: 0.0274839\ttotal: 30.3s\tremaining: 12.8s\n",
            "703:\tlearn: 0.0274677\ttotal: 30.3s\tremaining: 12.7s\n",
            "704:\tlearn: 0.0274487\ttotal: 30.4s\tremaining: 12.7s\n",
            "705:\tlearn: 0.0274315\ttotal: 30.4s\tremaining: 12.7s\n",
            "706:\tlearn: 0.0273997\ttotal: 30.5s\tremaining: 12.6s\n",
            "707:\tlearn: 0.0273851\ttotal: 30.5s\tremaining: 12.6s\n",
            "708:\tlearn: 0.0273601\ttotal: 30.5s\tremaining: 12.5s\n",
            "709:\tlearn: 0.0273354\ttotal: 30.6s\tremaining: 12.5s\n",
            "710:\tlearn: 0.0272797\ttotal: 30.6s\tremaining: 12.4s\n",
            "711:\tlearn: 0.0272388\ttotal: 30.7s\tremaining: 12.4s\n",
            "712:\tlearn: 0.0272049\ttotal: 30.7s\tremaining: 12.4s\n",
            "713:\tlearn: 0.0271606\ttotal: 30.8s\tremaining: 12.3s\n",
            "714:\tlearn: 0.0271204\ttotal: 30.8s\tremaining: 12.3s\n",
            "715:\tlearn: 0.0270841\ttotal: 30.8s\tremaining: 12.2s\n",
            "716:\tlearn: 0.0270640\ttotal: 30.9s\tremaining: 12.2s\n",
            "717:\tlearn: 0.0270507\ttotal: 30.9s\tremaining: 12.1s\n",
            "718:\tlearn: 0.0270062\ttotal: 31s\tremaining: 12.1s\n",
            "719:\tlearn: 0.0269874\ttotal: 31s\tremaining: 12.1s\n",
            "720:\tlearn: 0.0269679\ttotal: 31.1s\tremaining: 12s\n",
            "721:\tlearn: 0.0269498\ttotal: 31.1s\tremaining: 12s\n",
            "722:\tlearn: 0.0269219\ttotal: 31.1s\tremaining: 11.9s\n",
            "723:\tlearn: 0.0268854\ttotal: 31.2s\tremaining: 11.9s\n",
            "724:\tlearn: 0.0268539\ttotal: 31.2s\tremaining: 11.8s\n",
            "725:\tlearn: 0.0268059\ttotal: 31.3s\tremaining: 11.8s\n",
            "726:\tlearn: 0.0267482\ttotal: 31.3s\tremaining: 11.8s\n",
            "727:\tlearn: 0.0266798\ttotal: 31.4s\tremaining: 11.7s\n",
            "728:\tlearn: 0.0266556\ttotal: 31.4s\tremaining: 11.7s\n",
            "729:\tlearn: 0.0266177\ttotal: 31.4s\tremaining: 11.6s\n",
            "730:\tlearn: 0.0266002\ttotal: 31.5s\tremaining: 11.6s\n",
            "731:\tlearn: 0.0265912\ttotal: 31.5s\tremaining: 11.5s\n",
            "732:\tlearn: 0.0265619\ttotal: 31.6s\tremaining: 11.5s\n",
            "733:\tlearn: 0.0265408\ttotal: 31.6s\tremaining: 11.5s\n",
            "734:\tlearn: 0.0265180\ttotal: 31.7s\tremaining: 11.4s\n",
            "735:\tlearn: 0.0265099\ttotal: 31.7s\tremaining: 11.4s\n",
            "736:\tlearn: 0.0264706\ttotal: 31.7s\tremaining: 11.3s\n",
            "737:\tlearn: 0.0264510\ttotal: 31.8s\tremaining: 11.3s\n",
            "738:\tlearn: 0.0264096\ttotal: 31.8s\tremaining: 11.2s\n",
            "739:\tlearn: 0.0263697\ttotal: 31.9s\tremaining: 11.2s\n",
            "740:\tlearn: 0.0263373\ttotal: 31.9s\tremaining: 11.2s\n",
            "741:\tlearn: 0.0262948\ttotal: 32s\tremaining: 11.1s\n",
            "742:\tlearn: 0.0262765\ttotal: 32s\tremaining: 11.1s\n",
            "743:\tlearn: 0.0262691\ttotal: 32s\tremaining: 11s\n",
            "744:\tlearn: 0.0262526\ttotal: 32.1s\tremaining: 11s\n",
            "745:\tlearn: 0.0262175\ttotal: 32.1s\tremaining: 10.9s\n",
            "746:\tlearn: 0.0262038\ttotal: 32.2s\tremaining: 10.9s\n",
            "747:\tlearn: 0.0261770\ttotal: 32.2s\tremaining: 10.9s\n",
            "748:\tlearn: 0.0261511\ttotal: 32.3s\tremaining: 10.8s\n",
            "749:\tlearn: 0.0261049\ttotal: 32.3s\tremaining: 10.8s\n",
            "750:\tlearn: 0.0260741\ttotal: 32.3s\tremaining: 10.7s\n",
            "751:\tlearn: 0.0260475\ttotal: 32.4s\tremaining: 10.7s\n",
            "752:\tlearn: 0.0260198\ttotal: 32.4s\tremaining: 10.6s\n",
            "753:\tlearn: 0.0259902\ttotal: 32.5s\tremaining: 10.6s\n",
            "754:\tlearn: 0.0259689\ttotal: 32.5s\tremaining: 10.6s\n",
            "755:\tlearn: 0.0259421\ttotal: 32.6s\tremaining: 10.5s\n",
            "756:\tlearn: 0.0259015\ttotal: 32.6s\tremaining: 10.5s\n",
            "757:\tlearn: 0.0258709\ttotal: 32.6s\tremaining: 10.4s\n",
            "758:\tlearn: 0.0258379\ttotal: 32.7s\tremaining: 10.4s\n",
            "759:\tlearn: 0.0257883\ttotal: 32.7s\tremaining: 10.3s\n",
            "760:\tlearn: 0.0257704\ttotal: 32.8s\tremaining: 10.3s\n",
            "761:\tlearn: 0.0257366\ttotal: 32.8s\tremaining: 10.3s\n",
            "762:\tlearn: 0.0257165\ttotal: 32.9s\tremaining: 10.2s\n",
            "763:\tlearn: 0.0256929\ttotal: 32.9s\tremaining: 10.2s\n",
            "764:\tlearn: 0.0256755\ttotal: 33s\tremaining: 10.1s\n",
            "765:\tlearn: 0.0256519\ttotal: 33s\tremaining: 10.1s\n",
            "766:\tlearn: 0.0256127\ttotal: 33s\tremaining: 10s\n",
            "767:\tlearn: 0.0255618\ttotal: 33.1s\tremaining: 9.99s\n",
            "768:\tlearn: 0.0255381\ttotal: 33.1s\tremaining: 9.95s\n",
            "769:\tlearn: 0.0255100\ttotal: 33.2s\tremaining: 9.9s\n",
            "770:\tlearn: 0.0254890\ttotal: 33.2s\tremaining: 9.86s\n",
            "771:\tlearn: 0.0254654\ttotal: 33.2s\tremaining: 9.82s\n",
            "772:\tlearn: 0.0254397\ttotal: 33.3s\tremaining: 9.77s\n",
            "773:\tlearn: 0.0254166\ttotal: 33.3s\tremaining: 9.73s\n",
            "774:\tlearn: 0.0253980\ttotal: 33.4s\tremaining: 9.69s\n",
            "775:\tlearn: 0.0253919\ttotal: 33.4s\tremaining: 9.64s\n",
            "776:\tlearn: 0.0253476\ttotal: 33.5s\tremaining: 9.6s\n",
            "777:\tlearn: 0.0253272\ttotal: 33.5s\tremaining: 9.56s\n",
            "778:\tlearn: 0.0253050\ttotal: 33.5s\tremaining: 9.52s\n",
            "779:\tlearn: 0.0252579\ttotal: 33.6s\tremaining: 9.47s\n",
            "780:\tlearn: 0.0252258\ttotal: 33.6s\tremaining: 9.43s\n",
            "781:\tlearn: 0.0251999\ttotal: 33.7s\tremaining: 9.39s\n",
            "782:\tlearn: 0.0251858\ttotal: 33.7s\tremaining: 9.34s\n",
            "783:\tlearn: 0.0251756\ttotal: 33.8s\tremaining: 9.3s\n",
            "784:\tlearn: 0.0251420\ttotal: 33.8s\tremaining: 9.26s\n",
            "785:\tlearn: 0.0251215\ttotal: 33.8s\tremaining: 9.21s\n",
            "786:\tlearn: 0.0250833\ttotal: 33.9s\tremaining: 9.17s\n",
            "787:\tlearn: 0.0250641\ttotal: 33.9s\tremaining: 9.12s\n",
            "788:\tlearn: 0.0250285\ttotal: 34s\tremaining: 9.09s\n",
            "789:\tlearn: 0.0250060\ttotal: 34s\tremaining: 9.04s\n",
            "790:\tlearn: 0.0249542\ttotal: 34.1s\tremaining: 9s\n",
            "791:\tlearn: 0.0249339\ttotal: 34.1s\tremaining: 8.96s\n",
            "792:\tlearn: 0.0248992\ttotal: 34.1s\tremaining: 8.91s\n",
            "793:\tlearn: 0.0248883\ttotal: 34.2s\tremaining: 8.87s\n",
            "794:\tlearn: 0.0248757\ttotal: 34.2s\tremaining: 8.83s\n",
            "795:\tlearn: 0.0248672\ttotal: 34.3s\tremaining: 8.79s\n",
            "796:\tlearn: 0.0248421\ttotal: 34.3s\tremaining: 8.74s\n",
            "797:\tlearn: 0.0248264\ttotal: 34.4s\tremaining: 8.7s\n",
            "798:\tlearn: 0.0247998\ttotal: 34.4s\tremaining: 8.66s\n",
            "799:\tlearn: 0.0247699\ttotal: 34.5s\tremaining: 8.62s\n",
            "800:\tlearn: 0.0247383\ttotal: 34.5s\tremaining: 8.57s\n",
            "801:\tlearn: 0.0247020\ttotal: 34.5s\tremaining: 8.53s\n",
            "802:\tlearn: 0.0246799\ttotal: 34.6s\tremaining: 8.48s\n",
            "803:\tlearn: 0.0246563\ttotal: 34.6s\tremaining: 8.44s\n",
            "804:\tlearn: 0.0246367\ttotal: 34.7s\tremaining: 8.4s\n",
            "805:\tlearn: 0.0246133\ttotal: 34.7s\tremaining: 8.36s\n",
            "806:\tlearn: 0.0245988\ttotal: 34.8s\tremaining: 8.31s\n",
            "807:\tlearn: 0.0245622\ttotal: 34.8s\tremaining: 8.27s\n",
            "808:\tlearn: 0.0245385\ttotal: 34.9s\tremaining: 8.23s\n",
            "809:\tlearn: 0.0245248\ttotal: 34.9s\tremaining: 8.19s\n",
            "810:\tlearn: 0.0245059\ttotal: 34.9s\tremaining: 8.14s\n",
            "811:\tlearn: 0.0244957\ttotal: 35s\tremaining: 8.1s\n",
            "812:\tlearn: 0.0244778\ttotal: 35s\tremaining: 8.06s\n",
            "813:\tlearn: 0.0244592\ttotal: 35.1s\tremaining: 8.01s\n",
            "814:\tlearn: 0.0244299\ttotal: 35.1s\tremaining: 7.97s\n",
            "815:\tlearn: 0.0244209\ttotal: 35.2s\tremaining: 7.93s\n",
            "816:\tlearn: 0.0244019\ttotal: 35.2s\tremaining: 7.88s\n",
            "817:\tlearn: 0.0243855\ttotal: 35.2s\tremaining: 7.84s\n",
            "818:\tlearn: 0.0243740\ttotal: 35.3s\tremaining: 7.8s\n",
            "819:\tlearn: 0.0243543\ttotal: 35.3s\tremaining: 7.75s\n",
            "820:\tlearn: 0.0243510\ttotal: 35.4s\tremaining: 7.71s\n",
            "821:\tlearn: 0.0243074\ttotal: 35.4s\tremaining: 7.67s\n",
            "822:\tlearn: 0.0242701\ttotal: 35.5s\tremaining: 7.62s\n",
            "823:\tlearn: 0.0242503\ttotal: 35.5s\tremaining: 7.58s\n",
            "824:\tlearn: 0.0242327\ttotal: 35.5s\tremaining: 7.54s\n",
            "825:\tlearn: 0.0242264\ttotal: 35.6s\tremaining: 7.49s\n",
            "826:\tlearn: 0.0242123\ttotal: 35.6s\tremaining: 7.45s\n",
            "827:\tlearn: 0.0241819\ttotal: 35.7s\tremaining: 7.41s\n",
            "828:\tlearn: 0.0241473\ttotal: 35.7s\tremaining: 7.37s\n",
            "829:\tlearn: 0.0241315\ttotal: 35.8s\tremaining: 7.32s\n",
            "830:\tlearn: 0.0241043\ttotal: 35.8s\tremaining: 7.28s\n",
            "831:\tlearn: 0.0240818\ttotal: 35.8s\tremaining: 7.24s\n",
            "832:\tlearn: 0.0240575\ttotal: 35.9s\tremaining: 7.19s\n",
            "833:\tlearn: 0.0240079\ttotal: 35.9s\tremaining: 7.15s\n",
            "834:\tlearn: 0.0239851\ttotal: 36s\tremaining: 7.11s\n",
            "835:\tlearn: 0.0239799\ttotal: 36s\tremaining: 7.06s\n",
            "836:\tlearn: 0.0239718\ttotal: 36.1s\tremaining: 7.02s\n",
            "837:\tlearn: 0.0239628\ttotal: 36.1s\tremaining: 6.98s\n",
            "838:\tlearn: 0.0239367\ttotal: 36.1s\tremaining: 6.93s\n",
            "839:\tlearn: 0.0238950\ttotal: 36.2s\tremaining: 6.89s\n",
            "840:\tlearn: 0.0238761\ttotal: 36.2s\tremaining: 6.85s\n",
            "841:\tlearn: 0.0238614\ttotal: 36.3s\tremaining: 6.8s\n",
            "842:\tlearn: 0.0238467\ttotal: 36.3s\tremaining: 6.76s\n",
            "843:\tlearn: 0.0238331\ttotal: 36.3s\tremaining: 6.72s\n",
            "844:\tlearn: 0.0237910\ttotal: 36.4s\tremaining: 6.67s\n",
            "845:\tlearn: 0.0237655\ttotal: 36.4s\tremaining: 6.63s\n",
            "846:\tlearn: 0.0237411\ttotal: 36.5s\tremaining: 6.59s\n",
            "847:\tlearn: 0.0237354\ttotal: 36.5s\tremaining: 6.54s\n",
            "848:\tlearn: 0.0237148\ttotal: 36.6s\tremaining: 6.5s\n",
            "849:\tlearn: 0.0236758\ttotal: 36.6s\tremaining: 6.46s\n",
            "850:\tlearn: 0.0236596\ttotal: 36.6s\tremaining: 6.42s\n",
            "851:\tlearn: 0.0236324\ttotal: 36.7s\tremaining: 6.37s\n",
            "852:\tlearn: 0.0236145\ttotal: 36.7s\tremaining: 6.33s\n",
            "853:\tlearn: 0.0235929\ttotal: 36.8s\tremaining: 6.29s\n",
            "854:\tlearn: 0.0235623\ttotal: 36.8s\tremaining: 6.24s\n",
            "855:\tlearn: 0.0235444\ttotal: 36.8s\tremaining: 6.2s\n",
            "856:\tlearn: 0.0235319\ttotal: 36.9s\tremaining: 6.16s\n",
            "857:\tlearn: 0.0235131\ttotal: 36.9s\tremaining: 6.11s\n",
            "858:\tlearn: 0.0234981\ttotal: 37s\tremaining: 6.07s\n",
            "859:\tlearn: 0.0234727\ttotal: 37s\tremaining: 6.03s\n",
            "860:\tlearn: 0.0234393\ttotal: 37.1s\tremaining: 5.99s\n",
            "861:\tlearn: 0.0234086\ttotal: 37.1s\tremaining: 5.94s\n",
            "862:\tlearn: 0.0233895\ttotal: 37.2s\tremaining: 5.9s\n",
            "863:\tlearn: 0.0233611\ttotal: 37.2s\tremaining: 5.86s\n",
            "864:\tlearn: 0.0233324\ttotal: 37.2s\tremaining: 5.81s\n",
            "865:\tlearn: 0.0233113\ttotal: 37.3s\tremaining: 5.77s\n",
            "866:\tlearn: 0.0233065\ttotal: 37.3s\tremaining: 5.72s\n",
            "867:\tlearn: 0.0232939\ttotal: 37.4s\tremaining: 5.68s\n",
            "868:\tlearn: 0.0232586\ttotal: 37.4s\tremaining: 5.64s\n",
            "869:\tlearn: 0.0232445\ttotal: 37.4s\tremaining: 5.59s\n",
            "870:\tlearn: 0.0232281\ttotal: 37.5s\tremaining: 5.55s\n",
            "871:\tlearn: 0.0231914\ttotal: 37.5s\tremaining: 5.51s\n",
            "872:\tlearn: 0.0231728\ttotal: 37.6s\tremaining: 5.47s\n",
            "873:\tlearn: 0.0231636\ttotal: 37.6s\tremaining: 5.42s\n",
            "874:\tlearn: 0.0231523\ttotal: 37.7s\tremaining: 5.38s\n",
            "875:\tlearn: 0.0231172\ttotal: 37.7s\tremaining: 5.34s\n",
            "876:\tlearn: 0.0231071\ttotal: 37.7s\tremaining: 5.29s\n",
            "877:\tlearn: 0.0230856\ttotal: 37.8s\tremaining: 5.25s\n",
            "878:\tlearn: 0.0230695\ttotal: 37.8s\tremaining: 5.21s\n",
            "879:\tlearn: 0.0230548\ttotal: 37.9s\tremaining: 5.16s\n",
            "880:\tlearn: 0.0230255\ttotal: 37.9s\tremaining: 5.12s\n",
            "881:\tlearn: 0.0230111\ttotal: 38s\tremaining: 5.08s\n",
            "882:\tlearn: 0.0229973\ttotal: 38s\tremaining: 5.04s\n",
            "883:\tlearn: 0.0229808\ttotal: 38s\tremaining: 4.99s\n",
            "884:\tlearn: 0.0229426\ttotal: 38.1s\tremaining: 4.95s\n",
            "885:\tlearn: 0.0229259\ttotal: 38.1s\tremaining: 4.91s\n",
            "886:\tlearn: 0.0228987\ttotal: 38.2s\tremaining: 4.86s\n",
            "887:\tlearn: 0.0228871\ttotal: 38.2s\tremaining: 4.82s\n",
            "888:\tlearn: 0.0228709\ttotal: 38.3s\tremaining: 4.78s\n",
            "889:\tlearn: 0.0228375\ttotal: 38.3s\tremaining: 4.73s\n",
            "890:\tlearn: 0.0228135\ttotal: 38.3s\tremaining: 4.69s\n",
            "891:\tlearn: 0.0227857\ttotal: 38.4s\tremaining: 4.65s\n",
            "892:\tlearn: 0.0227638\ttotal: 38.4s\tremaining: 4.6s\n",
            "893:\tlearn: 0.0227389\ttotal: 38.5s\tremaining: 4.56s\n",
            "894:\tlearn: 0.0227129\ttotal: 38.5s\tremaining: 4.52s\n",
            "895:\tlearn: 0.0226979\ttotal: 38.6s\tremaining: 4.47s\n",
            "896:\tlearn: 0.0226724\ttotal: 38.6s\tremaining: 4.43s\n",
            "897:\tlearn: 0.0226538\ttotal: 38.6s\tremaining: 4.39s\n",
            "898:\tlearn: 0.0226221\ttotal: 38.7s\tremaining: 4.34s\n",
            "899:\tlearn: 0.0225835\ttotal: 38.7s\tremaining: 4.3s\n",
            "900:\tlearn: 0.0225705\ttotal: 38.8s\tremaining: 4.26s\n",
            "901:\tlearn: 0.0225619\ttotal: 38.8s\tremaining: 4.22s\n",
            "902:\tlearn: 0.0225522\ttotal: 38.9s\tremaining: 4.17s\n",
            "903:\tlearn: 0.0225388\ttotal: 38.9s\tremaining: 4.13s\n",
            "904:\tlearn: 0.0225144\ttotal: 38.9s\tremaining: 4.09s\n",
            "905:\tlearn: 0.0224895\ttotal: 39s\tremaining: 4.04s\n",
            "906:\tlearn: 0.0224540\ttotal: 39s\tremaining: 4s\n",
            "907:\tlearn: 0.0224155\ttotal: 39.1s\tremaining: 3.96s\n",
            "908:\tlearn: 0.0223955\ttotal: 39.1s\tremaining: 3.92s\n",
            "909:\tlearn: 0.0223676\ttotal: 39.2s\tremaining: 3.87s\n",
            "910:\tlearn: 0.0223556\ttotal: 39.2s\tremaining: 3.83s\n",
            "911:\tlearn: 0.0223262\ttotal: 39.2s\tremaining: 3.79s\n",
            "912:\tlearn: 0.0223097\ttotal: 39.3s\tremaining: 3.74s\n",
            "913:\tlearn: 0.0222915\ttotal: 39.3s\tremaining: 3.7s\n",
            "914:\tlearn: 0.0222664\ttotal: 39.4s\tremaining: 3.66s\n",
            "915:\tlearn: 0.0222479\ttotal: 39.4s\tremaining: 3.61s\n",
            "916:\tlearn: 0.0222125\ttotal: 39.5s\tremaining: 3.57s\n",
            "917:\tlearn: 0.0221843\ttotal: 39.5s\tremaining: 3.53s\n",
            "918:\tlearn: 0.0221588\ttotal: 39.5s\tremaining: 3.48s\n",
            "919:\tlearn: 0.0221399\ttotal: 39.6s\tremaining: 3.44s\n",
            "920:\tlearn: 0.0221253\ttotal: 39.6s\tremaining: 3.4s\n",
            "921:\tlearn: 0.0221074\ttotal: 39.7s\tremaining: 3.36s\n",
            "922:\tlearn: 0.0220935\ttotal: 39.7s\tremaining: 3.31s\n",
            "923:\tlearn: 0.0220908\ttotal: 39.8s\tremaining: 3.27s\n",
            "924:\tlearn: 0.0220693\ttotal: 39.8s\tremaining: 3.23s\n",
            "925:\tlearn: 0.0220555\ttotal: 39.8s\tremaining: 3.18s\n",
            "926:\tlearn: 0.0220334\ttotal: 39.9s\tremaining: 3.14s\n",
            "927:\tlearn: 0.0220282\ttotal: 39.9s\tremaining: 3.1s\n",
            "928:\tlearn: 0.0219848\ttotal: 40s\tremaining: 3.05s\n",
            "929:\tlearn: 0.0219728\ttotal: 40s\tremaining: 3.01s\n",
            "930:\tlearn: 0.0219340\ttotal: 40.1s\tremaining: 2.97s\n",
            "931:\tlearn: 0.0219112\ttotal: 40.1s\tremaining: 2.92s\n",
            "932:\tlearn: 0.0218970\ttotal: 40.1s\tremaining: 2.88s\n",
            "933:\tlearn: 0.0218795\ttotal: 40.2s\tremaining: 2.84s\n",
            "934:\tlearn: 0.0218600\ttotal: 40.2s\tremaining: 2.8s\n",
            "935:\tlearn: 0.0218512\ttotal: 40.3s\tremaining: 2.75s\n",
            "936:\tlearn: 0.0218465\ttotal: 40.3s\tremaining: 2.71s\n",
            "937:\tlearn: 0.0218392\ttotal: 40.3s\tremaining: 2.67s\n",
            "938:\tlearn: 0.0218253\ttotal: 40.4s\tremaining: 2.62s\n",
            "939:\tlearn: 0.0217927\ttotal: 40.4s\tremaining: 2.58s\n",
            "940:\tlearn: 0.0217717\ttotal: 40.5s\tremaining: 2.54s\n",
            "941:\tlearn: 0.0217499\ttotal: 40.5s\tremaining: 2.49s\n",
            "942:\tlearn: 0.0217315\ttotal: 40.6s\tremaining: 2.45s\n",
            "943:\tlearn: 0.0216906\ttotal: 40.6s\tremaining: 2.41s\n",
            "944:\tlearn: 0.0216680\ttotal: 40.6s\tremaining: 2.37s\n",
            "945:\tlearn: 0.0216453\ttotal: 40.7s\tremaining: 2.32s\n",
            "946:\tlearn: 0.0216239\ttotal: 40.7s\tremaining: 2.28s\n",
            "947:\tlearn: 0.0215952\ttotal: 40.8s\tremaining: 2.24s\n",
            "948:\tlearn: 0.0215750\ttotal: 40.8s\tremaining: 2.19s\n",
            "949:\tlearn: 0.0215490\ttotal: 40.9s\tremaining: 2.15s\n",
            "950:\tlearn: 0.0215231\ttotal: 40.9s\tremaining: 2.11s\n",
            "951:\tlearn: 0.0215007\ttotal: 41s\tremaining: 2.06s\n",
            "952:\tlearn: 0.0214881\ttotal: 41s\tremaining: 2.02s\n",
            "953:\tlearn: 0.0214522\ttotal: 41s\tremaining: 1.98s\n",
            "954:\tlearn: 0.0214278\ttotal: 41.1s\tremaining: 1.94s\n",
            "955:\tlearn: 0.0213884\ttotal: 41.1s\tremaining: 1.89s\n",
            "956:\tlearn: 0.0213694\ttotal: 41.2s\tremaining: 1.85s\n",
            "957:\tlearn: 0.0213568\ttotal: 41.2s\tremaining: 1.81s\n",
            "958:\tlearn: 0.0213278\ttotal: 41.3s\tremaining: 1.76s\n",
            "959:\tlearn: 0.0213088\ttotal: 41.3s\tremaining: 1.72s\n",
            "960:\tlearn: 0.0212876\ttotal: 41.3s\tremaining: 1.68s\n",
            "961:\tlearn: 0.0212802\ttotal: 41.4s\tremaining: 1.63s\n",
            "962:\tlearn: 0.0212585\ttotal: 41.4s\tremaining: 1.59s\n",
            "963:\tlearn: 0.0212366\ttotal: 41.5s\tremaining: 1.55s\n",
            "964:\tlearn: 0.0212145\ttotal: 41.5s\tremaining: 1.5s\n",
            "965:\tlearn: 0.0211843\ttotal: 41.6s\tremaining: 1.46s\n",
            "966:\tlearn: 0.0211560\ttotal: 41.6s\tremaining: 1.42s\n",
            "967:\tlearn: 0.0211428\ttotal: 41.6s\tremaining: 1.38s\n",
            "968:\tlearn: 0.0211103\ttotal: 41.7s\tremaining: 1.33s\n",
            "969:\tlearn: 0.0210892\ttotal: 41.7s\tremaining: 1.29s\n",
            "970:\tlearn: 0.0210892\ttotal: 41.8s\tremaining: 1.25s\n",
            "971:\tlearn: 0.0210739\ttotal: 41.8s\tremaining: 1.2s\n",
            "972:\tlearn: 0.0210227\ttotal: 41.8s\tremaining: 1.16s\n",
            "973:\tlearn: 0.0209824\ttotal: 41.9s\tremaining: 1.12s\n",
            "974:\tlearn: 0.0209699\ttotal: 41.9s\tremaining: 1.07s\n",
            "975:\tlearn: 0.0209568\ttotal: 42s\tremaining: 1.03s\n",
            "976:\tlearn: 0.0209285\ttotal: 42s\tremaining: 989ms\n",
            "977:\tlearn: 0.0209285\ttotal: 42.1s\tremaining: 946ms\n",
            "978:\tlearn: 0.0209093\ttotal: 42.1s\tremaining: 903ms\n",
            "979:\tlearn: 0.0209014\ttotal: 42.2s\tremaining: 860ms\n",
            "980:\tlearn: 0.0209014\ttotal: 42.2s\tremaining: 817ms\n",
            "981:\tlearn: 0.0208954\ttotal: 42.2s\tremaining: 774ms\n",
            "982:\tlearn: 0.0208730\ttotal: 42.3s\tremaining: 731ms\n",
            "983:\tlearn: 0.0208730\ttotal: 42.3s\tremaining: 688ms\n",
            "984:\tlearn: 0.0208348\ttotal: 42.3s\tremaining: 645ms\n",
            "985:\tlearn: 0.0208199\ttotal: 42.4s\tremaining: 602ms\n",
            "986:\tlearn: 0.0207905\ttotal: 42.4s\tremaining: 559ms\n",
            "987:\tlearn: 0.0207858\ttotal: 42.5s\tremaining: 516ms\n",
            "988:\tlearn: 0.0207469\ttotal: 42.5s\tremaining: 473ms\n",
            "989:\tlearn: 0.0207291\ttotal: 42.6s\tremaining: 430ms\n",
            "990:\tlearn: 0.0207291\ttotal: 42.6s\tremaining: 387ms\n",
            "991:\tlearn: 0.0206864\ttotal: 42.6s\tremaining: 344ms\n",
            "992:\tlearn: 0.0206864\ttotal: 42.7s\tremaining: 301ms\n",
            "993:\tlearn: 0.0206783\ttotal: 42.7s\tremaining: 258ms\n",
            "994:\tlearn: 0.0206561\ttotal: 42.7s\tremaining: 215ms\n",
            "995:\tlearn: 0.0206426\ttotal: 42.8s\tremaining: 172ms\n",
            "996:\tlearn: 0.0206114\ttotal: 42.8s\tremaining: 129ms\n",
            "997:\tlearn: 0.0206009\ttotal: 42.9s\tremaining: 85.9ms\n",
            "998:\tlearn: 0.0205807\ttotal: 42.9s\tremaining: 43ms\n",
            "999:\tlearn: 0.0205646\ttotal: 42.9s\tremaining: 0us\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ue0Oz11od2r_",
        "outputId": "29b10917-4d7e-42d3-ae67-708d248d41bb"
      },
      "source": [
        "joblib.dump(cb_clf,'cb_clf1.pkl')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['cb_clf1.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uau9zOeXeAwu",
        "outputId": "f1edd8f8-9f56-47ff-d9a7-7e5e0dcf3e29"
      },
      "source": [
        "cb_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 1, ..., 0, 1, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tOdRl9Vnd2yb",
        "outputId": "0942fcad-a8cb-419d-a52e-ad0517bf40c7"
      },
      "source": [
        "cb_acc = accuracy_score(y_valid, cb_pred)\r\n",
        "cb_acc\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9878942785720305"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "5gh6V3H7yc54",
        "outputId": "30ca097f-6277-41cb-df15-2b900bf80a75"
      },
      "source": [
        "cb_valid_pred = pd.DataFrame(cb_pred, columns =['catboost'])\r\n",
        "cb_valid_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>catboost</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35598</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35599</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35600</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35601</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35602</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35603 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       catboost\n",
              "0             0\n",
              "1             0\n",
              "2             1\n",
              "3             0\n",
              "4             1\n",
              "...         ...\n",
              "35598         0\n",
              "35599         0\n",
              "35600         0\n",
              "35601         1\n",
              "35602         1\n",
              "\n",
              "[35603 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYwAFoHERerA"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "cb_valid_pred.to_csv(\"cb_valid_pred1.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rkI9-l1kdqhb"
      },
      "source": [
        "cb_params = {\r\n",
        "    'n_estimators' : [100, 500, 1000],\r\n",
        "    'learning_rate' : [0.001, 0.01, 0.1],\r\n",
        "}\r\n",
        "\r\n",
        "\r\n",
        "grid_cv = GridSearchCV(cb_clf, param_grid = cb_params, cv = 3)\r\n",
        "grid_cv.fit(X_train, y_train)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', grid_cv.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(grid_cv.best_score_))\r\n",
        "\r\n",
        "- 최적 하이퍼 파라미터:\r\n",
        " {'learning_rate': 0.1, 'n_estimators': 100}\r\n",
        "  최고 예측 정확도: 0.887512\r\n",
        "\r\n",
        "\r\n",
        "- valid - acc = 0.9797607318789585"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aVIzTkFySP9x"
      },
      "source": [
        "### ExtraTree\r\n",
        "\r\n",
        "- gridCV를 사용했을 때, 오히려 과적합되는 것 같음.(이번 경우에만)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hpeEQOBOSUD-",
        "outputId": "c8d26284-7d70-4508-8ed5-a16e40fa4a6b"
      },
      "source": [
        "ext_clf =  ExtraTreesClassifier(random_state=42)\r\n",
        "ext_clf.fit(X_train, y_train)\r\n",
        "\r\n",
        "#default n_estimators = 100, max_depth=None"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
              "                     criterion='gini', max_depth=None, max_features='auto',\n",
              "                     max_leaf_nodes=None, max_samples=None,\n",
              "                     min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                     min_samples_leaf=1, min_samples_split=2,\n",
              "                     min_weight_fraction_leaf=0.0, n_estimators=100,\n",
              "                     n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
              "                     warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DmZTN5KRU1AC",
        "outputId": "9b8a9664-788c-487e-df51-75e05a357ffe"
      },
      "source": [
        "\r\n",
        "ext_pred = ext_clf.predict(X_valid)\r\n",
        "ext_acc = accuracy_score(y_valid, ext_pred)\r\n",
        "ext_acc\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9769401454933573"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jXfFxMnESZB1",
        "outputId": "8b844661-d54b-4c6c-aa13-d971499f19f9"
      },
      "source": [
        "ext_params = {\r\n",
        "    'n_estimators' : [100, 1000]\r\n",
        "}\r\n",
        "\r\n",
        "\r\n",
        "ext_cv = GridSearchCV(ext_clf, param_grid = ext_params, cv = 3)\r\n",
        "ext_cv.fit(X_train, y_train)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', ext_cv.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(ext_cv.best_score_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'n_estimators': 1000}\n",
            "최고 예측 정확도: 0.891168\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NExyB_TWYvfj",
        "outputId": "e87b3491-ecf4-4498-f481-29f596c2236d"
      },
      "source": [
        "ext_cv.get_params"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<bound method BaseEstimator.get_params of GridSearchCV(cv=3, error_score=nan,\n",
              "             estimator=ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0,\n",
              "                                            class_weight=None, criterion='gini',\n",
              "                                            max_depth=None, max_features='auto',\n",
              "                                            max_leaf_nodes=None,\n",
              "                                            max_samples=None,\n",
              "                                            min_impurity_decrease=0.0,\n",
              "                                            min_impurity_split=None,\n",
              "                                            min_samples_leaf=1,\n",
              "                                            min_samples_split=2,\n",
              "                                            min_weight_fraction_leaf=0.0,\n",
              "                                            n_estimators=100, n_jobs=None,\n",
              "                                            oob_score=False, random_state=42,\n",
              "                                            verbose=0, warm_start=False),\n",
              "             iid='deprecated', n_jobs=None,\n",
              "             param_grid={'n_estimators': [100, 500, 1000]},\n",
              "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
              "             scoring=None, verbose=0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1H-3soGzdpOe",
        "outputId": "00b3eec9-ef8e-4b17-b763-cfa3ebaa81e7"
      },
      "source": [
        "joblib.dump(ext_cv,'ext_cv1.pkl')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ext_cv1.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WK15o5-3SZGC",
        "outputId": "fbce8f81-b04f-4731-aec9-5ae82ada75f5"
      },
      "source": [
        "# predict\r\n",
        "ext_pred = ext_cv.best_estimator_.predict(X_valid)\r\n",
        "\r\n",
        "ext_acc = accuracy_score(y_valid, ext_pred)\r\n",
        "ext_acc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9771086706176446"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "qqbM6zXvSctD",
        "outputId": "d6718ca5-77e4-4202-9fa7-9aaf7fb00d37"
      },
      "source": [
        "ext_valid_pred = pd.DataFrame(ext_pred, columns =['ExtraTree'])\r\n",
        "ext_valid_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ExtraTree</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35598</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35599</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35600</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35601</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35602</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35603 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       ExtraTree\n",
              "0              0\n",
              "1              0\n",
              "2              1\n",
              "3              0\n",
              "4              1\n",
              "...          ...\n",
              "35598          0\n",
              "35599          0\n",
              "35600          0\n",
              "35601          1\n",
              "35602          1\n",
              "\n",
              "[35603 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6d0ZgwKhSh4s"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "ext_valid_pred.to_csv(\"ext_valid_pred1.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vE2LIzwUWU7F"
      },
      "source": [
        "### cv기반 스태킹\r\n",
        "- Kfold\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3TIhX9mWT1K"
      },
      "source": [
        "from sklearn.model_selection import KFold\r\n",
        "from sklearn.metrics import mean_absolute_error\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "# 개별 기반 모델에서 최종 모델이 사용할 train, test 데이터를 생성하기 위한 함수. \r\n",
        "def get_stacking_base_datasets(model, X_train_n, y_train_n, X_test_n, n_folds ):\r\n",
        "\r\n",
        "    kf = KFold(n_splits=n_folds, shuffle=False, random_state=42)\r\n",
        "    #추후에 메타 모델이 사용할 학습 데이터 반환을 위한 넘파이 배열 초기화 \r\n",
        "    train_fold_pred = np.zeros((X_train_n.shape[0] ,1 ))\r\n",
        "    test_pred = np.zeros((X_test_n.shape[0],n_folds))\r\n",
        "    print(model.__class__.__name__ , ' model 시작 ')\r\n",
        "    \r\n",
        "    for folder_counter , (train_index, valid_index) in enumerate(kf.split(X_train_n)):\r\n",
        "        #입력된 학습 데이터에서 기반 모델이 학습/예측할 폴드 데이터 셋 추출 \r\n",
        "        print('\\t 폴드 세트: ',folder_counter,' 시작 ')\r\n",
        "        X_tr = X_train_n[train_index] \r\n",
        "        y_tr = y_train_n[train_index] \r\n",
        "        X_te = X_train_n[valid_index]  \r\n",
        "        \r\n",
        "        \r\n",
        "        model.fit(X_tr , y_tr)       \r\n",
        "        \r\n",
        "        train_fold_pred[valid_index, :] = model.predict(X_te).reshape(-1,1)\r\n",
        "       \r\n",
        "        test_pred[:, folder_counter] = model.predict(X_test_n)\r\n",
        "            \r\n",
        "    # 폴드 세트 내에서 원본 테스트 데이터를 예측한 데이터를 평균하여 테스트 데이터로 생성 \r\n",
        "    test_pred_mean = np.mean(test_pred, axis=1).reshape(-1,1)    \r\n",
        "    \r\n",
        "    #train_fold_pred는 최종 메타 모델이 사용하는 학습 데이터, test_pred_mean은 테스트 데이터\r\n",
        "    return train_fold_pred , test_pred_mean"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "mWuiXIgTY_L0",
        "outputId": "04ed7e4f-f1fe-40d7-acff-2725d986e250"
      },
      "source": [
        "\r\n",
        "\r\n",
        "# gb, ada, cb, ext, lstm \r\n",
        "\r\n",
        "gb_train, gb_test = get_stacking_base_datasets(gb_clf, X_train, y_train, X_valid, 5)\r\n",
        "ada_train, ada_test = get_stacking_base_datasets(ada_clf, X_train, y_train, X_valid, 5)\r\n",
        "cb_train, cb_test = get_stacking_base_datasets(cb_clf, X_train, y_train, X_valid, 5)\r\n",
        "ext_train, ext_test = get_stacking_base_datasets(ext_clf, X_train, y_train, X_valid,  5)    \r\n",
        "lstm_train, lstm_test = get_stacking_base_datasets(lstm_clf, X_train, y_train, X_valid,  5)    \r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-75d452bff31c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# gb, ada, cb, ext, lstm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mgb_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgb_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_stacking_base_datasets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgb_clf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_valid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mada_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mada_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_stacking_base_datasets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mada_clf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_valid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcb_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_stacking_base_datasets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcb_clf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_valid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'gb_clf' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SvlkD1saZfcF"
      },
      "source": [
        "- StratifiedKFold"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a2Vqd61JSdLQ"
      },
      "source": [
        "### LSTM 모델링"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IOrznz_nNk33"
      },
      "source": [
        "# EDA기반으로 만든 feature 예측변수로 추가\r\n",
        "from keras import regularizers\r\n",
        "\r\n",
        "def LSTM_add_feature(vocab_size, embedding_size = 100, max_len=100):\r\n",
        "  input1 = keras.layers.Input(shape = [max_len,]) #문장 단어 input\r\n",
        "  input2 = keras.layers.Input(shape = [feature_num,]) # EDA기반 feature input\r\n",
        "\r\n",
        "  # LSTM\r\n",
        "  embedding = keras.layers.Embedding(vocab_size, embedding_size, weights = [embedding_matrix], input_length = max_len)(input1) # 임베딩 가중치 적용\r\n",
        "  dropout1 = keras.layers.SpatialDropout1D(0.3)(embedding)\r\n",
        "  lstm1 = keras.layers.LSTM(32, return_sequences = True)(dropout1)\r\n",
        "  lstm2 = keras.layers.LSTM(32)(lstm1)\r\n",
        "  dropout2 = keras.layers.Dropout(0.5)(lstm2)\r\n",
        "  #lstm_output = keras.layers.Dense(16, activation = \"selu\")(dropout2)\r\n",
        "\r\n",
        "  # MLP\r\n",
        "  concat = keras.layers.concatenate([dropout2,input2])\r\n",
        "  hidden = keras.layers.Dense(32, activation = \"selu\")(concat)\r\n",
        "  output = keras.layers.Dense(1, activation = \"sigmoid\")(hidden)\r\n",
        "\r\n",
        "  model = keras.Model(inputs = [input1, input2], outputs = [output])\r\n",
        "\r\n",
        "  model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate), loss=\"binary_crossentropy\", metrics = \"accuracy\")\r\n",
        "  model.summary()\r\n",
        "  return model\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vLyta-R27x7b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "406c4ded-51ba-48f1-b581-e9ad2b0995c2"
      },
      "source": [
        "# 훈련 시\r\n",
        "tf.random.set_seed(42)\r\n",
        "\r\n",
        "\r\n",
        "checkpoint_cb = keras.callbacks.ModelCheckpoint(\"lstm_stacking_valid_model.h5\",\r\n",
        "                                               save_best_only = True)\r\n",
        "\r\n",
        "# 하이퍼파라미터\r\n",
        "max_epoch = 5\r\n",
        "batch_size = 32\r\n",
        "learning_rate = 0.001\r\n",
        "\r\n",
        "model = LSTM_add_feature(vocab_size, max_len = max_len, embedding_size = embedding_size)\r\n",
        "history = model.fit(x=[X_train[:,:max_len],X_train[:,-feature_num:]], y=y_train, epochs=max_epoch,\r\n",
        "                batch_size = batch_size,  validation_data = ((X_valid[:,:max_len],X_valid[:,-feature_num:]),y_valid), validation_batch_size = batch_size,\r\n",
        "                 callbacks = [checkpoint_cb])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 20)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 20, 300)      10139400    input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout1d (SpatialDropo (None, 20, 300)      0           embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     (None, 20, 32)       42624       spatial_dropout1d[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   (None, 32)           8320        lstm[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 32)           0           lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, 4)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 36)           0           dropout[0][0]                    \n",
            "                                                                 input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 32)           1184        concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 1)            33          dense[0][0]                      \n",
            "==================================================================================================\n",
            "Total params: 10,191,561\n",
            "Trainable params: 10,191,561\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Epoch 1/5\n",
            "2597/2597 [==============================] - 344s 131ms/step - loss: 0.0949 - accuracy: 0.9654 - val_loss: 0.0277 - val_accuracy: 0.9912\n",
            "Epoch 2/5\n",
            "2597/2597 [==============================] - 334s 129ms/step - loss: 0.0157 - accuracy: 0.9951 - val_loss: 0.0323 - val_accuracy: 0.9898\n",
            "Epoch 3/5\n",
            "2597/2597 [==============================] - 332s 128ms/step - loss: 0.0077 - accuracy: 0.9980 - val_loss: 0.0409 - val_accuracy: 0.9903\n",
            "Epoch 4/5\n",
            "2597/2597 [==============================] - 327s 126ms/step - loss: 0.0041 - accuracy: 0.9988 - val_loss: 0.0449 - val_accuracy: 0.9898\n",
            "Epoch 5/5\n",
            "2597/2597 [==============================] - 327s 126ms/step - loss: 0.0034 - accuracy: 0.9991 - val_loss: 0.0684 - val_accuracy: 0.9869\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JUSGWJvkDgiD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "outputId": "e307fba8-5696-457e-84dd-75b9acfc713a"
      },
      "source": [
        "best_model = keras.models.load_model(\"lstm_stacking_valid_model.h5\") # 저장된 모델 불러오기\r\n",
        "plt.plot(pd.DataFrame(history.history)) # 0.0249 0.0211"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fd970b54a90>,\n",
              " <matplotlib.lines.Line2D at 0x7fd970b54ac8>,\n",
              " <matplotlib.lines.Line2D at 0x7fd970b54828>,\n",
              " <matplotlib.lines.Line2D at 0x7fd970b54780>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 111
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYC0lEQVR4nO3dcYwc533e8eeZ3Tsej3dHSuVRUkTJVFHJqBzEsXxQXKhohNoGaLWV/kjaUIDrKHAjoK3aGAlaKG0huyoCNA1gtGnUuorjOk4bK4pbBLRLQzAcFQGCStUpthVLqgJatSyqEkkrEo8Uebzb3V//mNm92b3Z2z1y7/b46vsBBvvOvO/O/G5u95nZ2b1bR4QAAFe+bNwFAABGg0AHgEQQ6ACQCAIdABJBoANAIurj2vD+/fvj0KFD49o8AFyRnn322R9GxHxV39gC/dChQ1pcXBzX5gHgimT7lX59XHIBgEQQ6ACQCAIdABJBoANAIgh0AEjEwEC3/QXbp2x/t0+/bf+67eO2n7N92+jLBAAMMswZ+hclHd6g/2OSbi6m+yX9x8svCwCwWQM/hx4Rf2T70AZD7pH0pcj/D+9TtvfZvi4iXh9RjUhBRDG18kmldmeKnttRjOtZPvQ45WPbtV/qfO+/p76cdY10vlzPiNbdq/Jfc2/j2L7/GbxqbL/BWzT2vYel6z/YZz2XbhR/WHS9pFdL8yeKZesC3fb9ys/ideONN45g0wlqtaTGsrR6QWpckFaXpdXzxbLz+XzjQt6/eqF7eb/7NC6uD8D2/IaBWWpfbrD2f3YBifP6RbPX7thAH1pEPCrpUUlaWFi4pGf4m//5izr92c9K9rrJUuVySf37LFlDLNfa2YiL27VJklr58naoKSSVw6wlR3lZOwBbcqy1O2eHLj0Myo+HzvLo7rOlLJNck7Na3s5q+eSssx9Kd+heYVhyfV3f2i/Ja9tpL2y3XS6wahtr7c76Outpb7+n7WK+t91uhnpqqVjHuhq6a1z/AOwztvTzufyzDt1216pd/l107aKe/bZuvrtOZ5UPjJ7xpTHZButvP97Xje1Z97ra27VkncegM+ePuSyTneVDsva8K9v5Uy2rHpcVV4a7+mpS5uI+7r5PscyddrZWn4vlnXWv1dpbb2fdtVpP3Wvb7Fqfi/W1t2uV+rrH1fbtU02jN4pAf03SDaX5g8WyLTH1vlt19X0/KzWbiuaK1GxIzdViakitVUVnWUOKRjEmn6LVkFrFfOk2Ou1mvrzVzKdmU9F5Ca6uoOh6NVUOFrUDJysFak1yTdFuq14syzpTaK0ttUM4y++TP8okOx+n/IGTH16KqXPpIBSK4rhSLOsbOOXmgIAaxTpGtB73ras8dP0BZeha2sq/5H7t0qGh6xvAomLsEO1QxTokqdU9PpoVl1GGWU/V+MtdR/EqLVqRv8pstfJ90W4r8vpbrfz5VDUuovs+pXb/yxxXpms/82lddeTIyNc7ikA/KukB249J+glJZ7by+vmextPac+5XpGh2dxQZOFBtlzQxJU1MS/UpaWJ3PtWL24mpUrvcV3WfYlm/9dXG9q9ygKRE6WRl3cGiVVzmqzoolPo67fY6qvo67Y36ygec9oGsVdlWRGXf7h9//5bsp4GJY/vLku6UtN/2CUmfljRR7OTPSTom6S5JxyWdl/RzW1Jp24/cJv3VT3UHcN9w7gnd+lR+xgzgiuLS5VPVan0vpL3bDfMpl3sH9IekfziyigY5dEc+AQC68JeiAJAIAh0AEkGgA0AiCHQASASBDgCJINABIBEEOgAkgkAHgEQQ6ACQCAIdABJBoANAIgh0AEgEgQ4AiSDQASARBDoAJIJAB4BEEOgAkAgCHQASQaADQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARBDoAJAIAh0AEkGgA0AiCHQASASBDgCJINABIBEEOgAkYqhAt33Y9ku2j9t+sKL/RttP2v6W7eds3zX6UgEAGxkY6LZrkh6R9DFJt0q61/atPcP+haTHI+IDko5I+g+jLhQAsLFhztBvl3Q8Il6OiBVJj0m6p2dMSJor2nsl/b/RlQgAGMYwgX69pFdL8yeKZWWfkfRx2yckHZP0j6pWZPt+24u2F0+fPn0J5QIA+hnVm6L3SvpiRByUdJek37G9bt0R8WhELETEwvz8/Ig2DQCQhgv01yTdUJo/WCwr+6SkxyUpIv6XpClJ+0dRIABgOMME+jOSbrZ9k+1J5W96Hu0Z8wNJH5Yk239ZeaBzTQUAttHAQI+IhqQHJD0h6UXln2Z53vbDtu8uhv2SpJ+3/R1JX5Z0X0TEVhUNAFivPsygiDim/M3O8rKHSu0XJN0x2tIAAJvBX4oCQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARBDoAJAIAh0AEkGgA0AiCHQASASBDgCJINABIBEEOgAkgkAHgEQQ6ACQCAIdABJBoANAIgh0AEgEgQ4AiSDQASARBDoAJIJAB4BEEOgAkAgCHQASQaADQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARAwV6LYP237J9nHbD/YZ83dsv2D7edu/O9oyAQCD1AcNsF2T9Iikj0o6IekZ20cj4oXSmJsl/bKkOyLiLdsHtqpgAEC1Yc7Qb5d0PCJejogVSY9JuqdnzM9LeiQi3pKkiDg12jIBAIMME+jXS3q1NH+iWFZ2i6RbbP+x7adsH65ake37bS/aXjx9+vSlVQwAqDSqN0Xrkm6WdKekeyX9pu19vYMi4tGIWIiIhfn5+RFtGgAgDRfor0m6oTR/sFhWdkLS0YhYjYj/K+nPlAc8AGCbDBPoz0i62fZNticlHZF0tGfMHyg/O5ft/covwbw8wjoBAAMMDPSIaEh6QNITkl6U9HhEPG/7Ydt3F8OekPSm7RckPSnpn0TEm1tVNABgPUfEWDa8sLAQi4uLY9k2AFypbD8bEQtVffylKAAkgkAHgEQQ6ACQCAIdABJBoANAIgh0AEgEgQ4AiSDQASARBDoAJIJAB4BEEOgAkAgCHQASQaADQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARBDoAJAIAh0AEkGgA0AiCHQASASBDgCJINABIBEEOgAkgkAHgEQQ6ACQCAIdABJBoANAIgh0AEgEgQ4AiRgq0G0ftv2S7eO2H9xg3E/ZDtsLoysRADCMgYFuuybpEUkfk3SrpHtt31oxblbSL0h6etRFAgAGG+YM/XZJxyPi5YhYkfSYpHsqxv0rSb8qaXmE9QEAhjRMoF8v6dXS/IliWYft2yTdEBH/Y6MV2b7f9qLtxdOnT2+6WABAf5f9pqjtTNJnJf3SoLER8WhELETEwvz8/OVuGgBQMkygvybphtL8wWJZ26ykH5X0P21/X9KHJB3ljVEA2F7DBPozkm62fZPtSUlHJB1td0bEmYjYHxGHIuKQpKck3R0Ri1tSMQCg0sBAj4iGpAckPSHpRUmPR8Tzth+2ffdWFwgAGE59mEERcUzSsZ5lD/UZe+fllwUA2Cz+UhQAEkGgA0AiCHQASASBDgCJINABIBEEOgAkgkAHgEQQ6ACQCAIdABJBoANAIgh0AEgEgQ4AiSDQASARBDoAJIJAB4BEEOgAkAgCHQASQaADQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARBDoAJAIAh0AEkGgA0AiCHQASASBDgCJINABIBEEOgAkgkAHgEQMFei2D9t+yfZx2w9W9P+i7RdsP2f7m7bfM/pSAQAbGRjotmuSHpH0MUm3SrrX9q09w74laSEifkzSVyT9m1EXCgDY2DBn6LdLOh4RL0fEiqTHJN1THhART0bE+WL2KUkHR1smAGCQYQL9ekmvluZPFMv6+aSkr1d12L7f9qLtxdOnTw9fJQBgoJG+KWr745IWJP1aVX9EPBoRCxGxMD8/P8pNA8C7Xn2IMa9JuqE0f7BY1sX2RyT9c0k/GREXR1MeAGBYw5yhPyPpZts32Z6UdETS0fIA2x+Q9J8k3R0Rp0ZfJgBgkIGBHhENSQ9IekLSi5Iej4jnbT9s++5i2K9JmpH0+7a/bfton9UBALbIMJdcFBHHJB3rWfZQqf2REdcFANgk/lIUABJBoANAIgh0AEgEgQ4AiSDQASARBDoAJIJAB4BEEOgAkAgCHQASQaADQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARBDoAJAIAh0AEkGgA0AiCHQASMRQ3ykKAKi20lzR2ZWza9Pq2e750vJzK+d0duWsPvG+T+jDN3545LUQ6ADetSJCFxoXdHblrM6t5mG7tLLUCd6zqz3zFaF9sXlxw21kzjQzMaPZyVnNTc5pZnJG2RZdHLniAv3t8ytaXm1p/8yk6jWuGAHvZq1o6Z3VdyrPhnuXnVs9p6WVpbX5IqQb0dhwG/WsrrnJOc1Ozmp2Ylazk7O6ds+1+XxpmpmY6YybmVxrT9enZXtb9scVF+i/v3hCv3LsRWWW9s/s0jVzU7pmbpcOzE3pmtm8nS/L21dNTyrLtmdnAtic1daqzq2c07mVc1pa7Q7bdviWz5zLQdzuC8WG29hd390J4tnJWV09dbXeM/eetfDtOXtuj5ubnNPMxIx21XZtWyBfrisu0H/yvfOa3lXTyTPLOrl0USfPLuu1t5f1rR+8rTffWVk3fqJmHZid0oG5XZ3APzA3pWtLoX9gbkpzU/Ur5pcGbJeIUCtaakVLzWh2tSNCzWiqGU2dWz3XFbRLK0udIF53GaN0Bn2hcWHD7VvuBG77zPe6mev03sn3di3vDeN2gM9Mzmgim9imvTV+V1yg33LNrG65Zrayb6XR0ulzF/XGmWWdWlrWyaVlnTx7USeXlnVq6aK+d/qc/vh7P9TZ5fUvsaYmsjzgZ6d0zd4pXTObn+kf6Dnjn5684nbZu1qz1dRqa1WrrVU1Wo1Oe7XZM1/ub67mwaUixFp5uxUtNVtNhWLj5RUB2BuCg5Zfzn2r7n+p6xx09jtI3fVOsLbDdn56viuMe0O4PO2Z2KPMXFodVlLpNFnPdP2+3bp+3+4Nx51faejU0sVO4LfD/41i2Z+eeFvfWFrW8mpr3X1nd9XzwC/O+A/MlS/z5Lfzs7u0q17bqh9zbCKiOgSbq1qNDUKyT385QHsDd6O+YfpXm6tqREOtWP873G6ZM2XKlDlTLavJsmquKctKy12TXSx31pmqlveOqWd1TXpyqPtW3b9q2WbvOzMx03XduH0pY3d9N698t1FSgT6s6cm6Du2v69D+PX3HRITOXmzo5JkLen3pgt44845eX7qgU2fP69TSBZ06e0Yvv/KGfnhuWY1WU3JLUkhuyQrtna7p6j11Xb2nrquKad90e6pp73Rde3ZlskPNVrNzZtSe2md95TOr3jO4Zqu5rq995lieb0azKwD7huOA/kZr4zePLkfNNU1kE6pndU1kE/lUm1i3rJ7VNZlNanpiumtZ5z7t+dpEdX9tQnUP6M/qa8GlTFlWBFmpbVm1bP2YdcudEWjYNldcoH/zB9/UV7/31XUBti4QW/2DsHy/9llc15hSkPZ9ybknn6auqe5uSDpVTFoupje3YIcUOmd9xVlg+eyp5lplkLXDbLo+XRmeXfMb9V9meNZdVy1L7xUNsN2uuEBfurikV5ZeWQuv4qVre34ym8zDrLSs721Wq+7b4L71rL7u5WdvkHbGut5ZHmG9c7GlM+cbevt8s7ht6K13mvrzd1b15jureuudps6cbypkKTKpuK1nNe2fmdKBmd06MLtbB+by22tnp3Xt3j26bm63rtm7mzd2gXc5R1zemx6XamFhIRYXF8ey7Z3sYqOp02cv6uRSfm3/jaXlTvvk2eKTPUvLlW/s7p6orX2Ec27tjd39s5OarNU0UbMm6pkma5nqWaldsyZq3e18ytv1zBwogB3C9rMRsVDVN9QZuu3Dkv6dpJqkz0fEv+7p3yXpS5I+qPzCws9ExPcvp+h3q131mg5eNa2DV01vOO78SqMT7u1P8ZQ/1fPcibd1ss8bu5eiX9hP9CyrFweG3uUTtazoq25P1KzJetY5gJTbgw5CExUHJQ5CeDcaGOi2a5IekfRRSSckPWP7aES8UBr2SUlvRcRfsn1E0q9K+pmtKBi56cm6btpf100D3thdWm7ozXMXtdoMrTZbWmm21Kho51NFu9HSamut3WiFVvq0V5strTRaurDa1NJy3l5tFn2NllaaoUarWGczv+9W6j0ITRYHnKqDTW/bljJbWXHrUjvLJNuy1sbk/cWYzJ37b2ZM1zaznm06f5/EpRrW1WVJWl9n98+Rr6e9jXV1lcdkfWov1Vr+GVSqsZjt7Kf2OuV2uxjT09+5b2cd7ozvGsvButIwZ+i3SzoeES9Lku3HJN0jqRzo90j6TNH+iqTfsO0Y1/UcSMof9Ht3T2jv7p35hxURoWYr8oNHKej7HWAaxUGoqp0foKrbG62z3W4fhNrLWhFSSK0ItYrb6Mzny6LndqMx7T6MXmXgtw8eXWPWHxzUe/CpWFd70Frf+nV11dJvW8WqLOlTH7lFf+v9PzLyfTFMoF8v6dXS/AlJP9FvTEQ0bJ+R9Bck/XAURSJNtvPLIzVpt94dn3IpB3z3AaAI/VZxqz5jWusPGlJpna3hDj4D6xhqTM/4/AfMf86iGcXP0u4vn+Ply6LUtzZf3l9V/VFaSVVfe75dTPRZV6nknlq7txPFfs5/pvXbKnq7t1X6ucvrV0j7prfmJGtbP+Vi+35J90vSjTfeuJ2bBnaE/I91pNraORswMsP8Te1rkm4ozR8sllWOsV2XtFcVn7qOiEcjYiEiFubn5y+tYgBApWEC/RlJN9u+yfakpCOSjvaMOSrpZ4v2T0v6Q66fA8D2GnjJpbgm/oCkJ5R/bPELEfG87YclLUbEUUm/Jel3bB+X9OfKQx8AsI2GuoYeEcckHetZ9lCpvSzpb4+2NADAZvB/KQEgEQQ6ACSCQAeARBDoAJCIsf23RdunJb1yiXffr535V6jUtTnUtXk7tTbq2pzLqes9EVH5hzxjC/TLYXux37+PHCfq2hzq2rydWht1bc5W1cUlFwBIBIEOAIm4UgP90XEX0Ad1bQ51bd5OrY26NmdL6roir6EDANa7Us/QAQA9CHQASMSODnTbh22/ZPu47Qcr+nfZ/r2i/2nbh3ZIXffZPm3728X097apri/YPmX7u336bfvXi7qfs33bDqnrTttnSvvroapxI67pBttP2n7B9vO2f6FizLbvryHrGsf+mrL9v21/p6jrX1aM2fbn45B1jeX5WGy7Zvtbtr9W0Tf6/RXFV0nttEn5v+r9nqS/KGlS0nck3doz5h9I+lzRPiLp93ZIXfdJ+o0x7LO/Juk2Sd/t03+XpK8r/1rDD0l6eofUdaekr23zvrpO0m1Fe1bSn1X8Hrd9fw1Z1zj2lyXNFO0JSU9L+lDPmHE8H4epayzPx2Lbvyjpd6t+X1uxv3byGXrny6kjYkVS+8upy+6R9NtF+yuSPuyt/zrwYeoai4j4I+X/j76feyR9KXJPSdpn+7odUNe2i4jXI+JPivZZSS8q/27csm3fX0PWte2KfXCumJ0opt5PVGz783HIusbC9kFJf0PS5/sMGfn+2smBXvXl1L0P7K4vp5bU/nLqcdclST9VvEz/iu0bKvrHYdjax+GvFC+bv277fdu54eKl7geUn92VjXV/bVCXNIb9VVw++LakU5K+ERF999c2Ph+HqUsaz/Px30r6p5JaffpHvr92cqBfyb4q6VBE/Jikb2jtKIxqf6L8/1O8X9K/l/QH27Vh2zOS/pukT0XE0nZtd5ABdY1lf0VEMyJ+XPn3Ct9u+0e3Y7uDDFHXtj8fbf9NSaci4tmt3lbZTg70kX059XbXFRFvRsTFYvbzkj64xTUNa5h9uu0iYqn9sjnyb8easL1/q7dre0J5aP7XiPjvFUPGsr8G1TWu/VXa/tuSnpR0uKdrHM/HgXWN6fl4h6S7bX9f+WXZv277v/SMGfn+2smBvlO/nHpgXT3XWe9Wfh10Jzgq6RPFpzc+JOlMRLw+7qJsX9u+dmj7duWPyy0NgmJ7vyXpxYj4bJ9h276/hqlrTPtr3va+or1b0kcl/Z+eYdv+fBymrnE8HyPilyPiYEQcUp4RfxgRH+8ZNvL9NdR3io5D7NAvpx6yrn9s+25JjaKu+7a6Lkmy/WXln4DYb/uEpE8rf5NIEfE55d8Le5ek45LOS/q5HVLXT0v6+7Ybki5IOrINB+Y7JP1dSX9aXH+VpH8m6cZSXePYX8PUNY79dZ2k37ZdU34AeTwivjbu5+OQdY3l+Vhlq/cXf/oPAInYyZdcAACbQKADQCIIdABIBIEOAIkg0AEgEQQ6ACSCQAeARPx/mOIJR+Cxr9kAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O3X-721afCH9",
        "outputId": "ef33bc25-6253-4fd7-a519-db04d08478bd"
      },
      "source": [
        "# 성능\r\n",
        "best_model.evaluate((X_valid[:,:-feature_num],X_valid[:,-feature_num:]),y_valid)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1113/1113 [==============================] - 8s 6ms/step - loss: 0.0277 - accuracy: 0.9912\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.02765706554055214, 0.9912086129188538]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IQYpSLq4GQxc",
        "outputId": "2125f3d1-8e7e-4e60-d41e-50e3846c0596"
      },
      "source": [
        "lstm_pred = best_model.predict((X_valid[:,:-feature_num],X_valid[:,-feature_num:]))\r\n",
        "lstm_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[5.9819221e-04],\n",
              "       [1.5429556e-03],\n",
              "       [9.9947447e-01],\n",
              "       ...,\n",
              "       [7.1025491e-03],\n",
              "       [9.9992704e-01],\n",
              "       [9.9984050e-01]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lYFhoyd7SXdm",
        "outputId": "d8b8269d-b5ec-4b4f-e393-48260bef143d"
      },
      "source": [
        "lstm_pred1 = lstm_pred.reshape(-1)\r\n",
        "lstm_pred1.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(35603,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "3TgG3c7MSdaE",
        "outputId": "21fc7046-d2dc-4adb-fac3-f8ba09e34aac"
      },
      "source": [
        "lstm_valid_pred = pd.DataFrame(lstm_pred1, columns =['LSTM'])\r\n",
        "lstm_valid_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>LSTM</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.000598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.001543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.999474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.001084</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.999927</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35598</th>\n",
              "      <td>0.000387</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35599</th>\n",
              "      <td>0.009572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35600</th>\n",
              "      <td>0.007103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35601</th>\n",
              "      <td>0.999927</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35602</th>\n",
              "      <td>0.999840</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35603 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           LSTM\n",
              "0      0.000598\n",
              "1      0.001543\n",
              "2      0.999474\n",
              "3      0.001084\n",
              "4      0.999927\n",
              "...         ...\n",
              "35598  0.000387\n",
              "35599  0.009572\n",
              "35600  0.007103\n",
              "35601  0.999927\n",
              "35602  0.999840\n",
              "\n",
              "[35603 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JMhQSTLuSxAF"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "lstm_valid_pred.to_csv(\"lstm_valid_pred1.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FPKiOiLAenlg"
      },
      "source": [
        "### 스태킹\r\n",
        "\r\n",
        "- 스태킹할때, gridCV를 사용하면 과적합되는 것 같음."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gbr5zVE_POKD",
        "outputId": "52bc7e9d-22b1-4429-9c3d-1ba499e1cdc0"
      },
      "source": [
        "# 아래 LSTM_add_feature()로 만든 예측값\r\n",
        "\r\n",
        "lstm_pred1 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([5.9819221e-04, 1.5429556e-03, 9.9947447e-01, ..., 7.1025491e-03,\n",
              "       9.9992704e-01, 9.9984050e-01], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LvYNOy7ofS_0",
        "outputId": "940d79e8-fbfc-426c-e83c-c28244ddcd64"
      },
      "source": [
        "all_pred = np.array([ada_pred, cb_pred, lstm_pred1])\r\n",
        "all_pred.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3, 35603)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 135
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a-HBWADSfhvC",
        "outputId": "b388f5b2-8fcf-487d-c60a-7d7962419b1a"
      },
      "source": [
        "all_pred = np.transpose(all_pred)\r\n",
        "all_pred.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(35603, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uSBH7Hs6fqwd",
        "outputId": "d115063a-0fca-4c3e-a465-7e470bdfdf7d"
      },
      "source": [
        "lgbm_clf=LGBMClassifier(random_state=42)\r\n",
        "lgbm_clf.fit(all_pred, y_valid)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
              "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
              "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
              "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
              "               random_state=42, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
              "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dk58Hqrjgw-2",
        "outputId": "f4071718-08d7-4df7-f705-c0651adb1b17"
      },
      "source": [
        "\r\n",
        "lgbm_pred = lgbm_clf.predict(all_pred)\r\n",
        "lgbm_acc = accuracy_score(y_valid, lgbm_pred)\r\n",
        "lgbm_acc\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9925006319692161"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "crpUCQpcfq0Z",
        "outputId": "4d5520b5-e649-4600-d3dd-408f8762b34e"
      },
      "source": [
        "lgbm_params={'boosting':['gbdt'],\r\n",
        "               'objective':['binary'],\r\n",
        "               'learning_rate':[0.001,0.05,0.1],\r\n",
        "               'max_depth':[2,4,6,8],\r\n",
        "               'n_estimators':[100,500,1000]}\r\n",
        "\r\n",
        "grid_lgbm = GridSearchCV(lgbm_clf,param_grid=lgbm_params,cv=3)\r\n",
        "grid_lgbm.fit(all_pred,y_valid)\r\n",
        "\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', grid_lgbm.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(grid_lgbm.best_score_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'boosting': 'gbdt', 'learning_rate': 0.05, 'max_depth': 2, 'n_estimators': 100, 'objective': 'binary'}\n",
            "최고 예측 정확도: 0.992107\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dozHxwVfczI4",
        "outputId": "f75a357b-67b0-440c-acc3-1f2ec5448909"
      },
      "source": [
        "joblib.dump(lgbm_clf,'lgbm_stacking_clf.pkl')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['lgbm_stacking_clf.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XzygWaiShTS0",
        "outputId": "efe0bbb2-3e95-4232-e449-fc65f2e476aa"
      },
      "source": [
        "stacking_pred=grid_lgbm.predict(all_pred)\r\n",
        "stacking_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 1, ..., 0, 1, 1], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Usc9DgLWh59f",
        "outputId": "f4fb337c-bc9d-4f7d-bb14-bb0858185893"
      },
      "source": [
        "stacking_acc = accuracy_score(y_valid, stacking_pred)\r\n",
        "stacking_acc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9922759318034997"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "5sjWjKIPh5Xp",
        "outputId": "cdf376a9-9774-4a7b-c6a1-68e31ff34b53"
      },
      "source": [
        "stacking_valid_pred = pd.DataFrame(ext_pred, columns =['gb, cb'])\r\n",
        "stacking_valid_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gb, cb</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35520</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35521</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35522</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35523</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35524</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35525 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       gb, cb\n",
              "0           1\n",
              "1           0\n",
              "2           1\n",
              "3           1\n",
              "4           1\n",
              "...       ...\n",
              "35520       1\n",
              "35521       0\n",
              "35522       1\n",
              "35523       1\n",
              "35524       1\n",
              "\n",
              "[35525 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 219
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CoUEhmK0jWsO"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "stacking_valid_pred.to_csv(\"stacking_valid_pred.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zzq4b5BrfYBQ"
      },
      "source": [
        "# 전체 train 스태킹 + test"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jaBckYhwiJR_"
      },
      "source": [
        "##### test셋 전처리"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V8AfnSdRiJmd"
      },
      "source": [
        "\r\n",
        "feature_num = 4\r\n",
        "test = pd.read_csv(\"/content/drive/MyDrive/Colab Notebooks/dacon_news/data/news_test.csv\")\r\n",
        "\r\n",
        "# EDA 기반 feature 추가\r\n",
        "test[\"info1_title\"]=test[\"title\"].apply(title_choose)\r\n",
        "test[\"info1_content\"]=test[\"content\"].apply(content_choose)\r\n",
        "test[\"content_startswith_[\"]=test.content.apply(lambda x : str(x).startswith(\"[\" ) or str(x).startswith(\"(\") or str(x).startswith(\"제목\"))+0\r\n",
        "title_group = (test.groupby([\"title\"]).count())[\"n_id\"]\r\n",
        "test[\"new_ord\"]=test.apply(lambda x: x[\"ord\"]/title_group[x[\"title\"]], axis=1)\r\n",
        "#test[\"info1_words\"]=test[\"content\"].apply(in_bull)\r\n",
        "\r\n",
        "# 전처리\r\n",
        "test['new_article'], okt = text_preprocessing(test['content'])\r\n",
        "test_X_seq = vectorizer.texts_to_sequences(test[\"new_article\"])\r\n",
        "test_X = pad_sequences(test_X_seq, maxlen = max_len, truncating=\"post\") # 길이를 맞춰줌\r\n",
        "\r\n",
        "# concat\r\n",
        "test_X = np.concatenate([test_X,test[[\"info1_title\",\"info1_content\",\"new_ord\",\"content_startswith_[\"]].values.reshape(-1,feature_num)], axis=1)"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lUHEzqOkERCj",
        "outputId": "4da37d0c-07f8-4311-cd1b-91e2065e3958"
      },
      "source": [
        "test_X.shape"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565, 24)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MbxuS9XGfi0-"
      },
      "source": [
        "##### GBM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 464
        },
        "id": "tM0pEwrrfqVD",
        "outputId": "526ea197-9323-4b0c-be3a-b84ba5057056"
      },
      "source": [
        "gb_clf = GradientBoostingClassifier(random_state = 42) # 기본값: max_depth=3, learning_rate=0.1\r\n",
        "\r\n",
        "gbm_params = {\r\n",
        "    'n_estimators' : [100, 1000],\r\n",
        "    'learning_rate' : [0.01, 0.1],\r\n",
        "}\r\n",
        "\r\n",
        "gb_grid_tst = GridSearchCV(gb_clf, param_grid = gbm_params, cv = 3)\r\n",
        "gb_grid_tst.fit(train_X, train_y)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', gb_grid_tst.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(gb_grid_tst.best_score_))\r\n",
        "\r\n",
        "joblib.dump(gb_grid_tst,'gb_grid_tst.pkl')\r\n",
        "\r\n",
        "gb_pred_tst = gb_grid_tst.best_estimator_.predict(test_X)\r\n",
        "\r\n",
        "gb_test_predict = pd.DataFrame(gb_pred_tst, columns =['GradientBoosting'])\r\n",
        "gb_test_predict"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'learning_rate': 0.1, 'n_estimators': 1000}\n",
            "최고 예측 정확도: 0.932118\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>GradientBoosting</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        GradientBoosting\n",
              "0                      0\n",
              "1                      0\n",
              "2                      0\n",
              "3                      0\n",
              "4                      0\n",
              "...                  ...\n",
              "142560                 1\n",
              "142561                 1\n",
              "142562                 1\n",
              "142563                 1\n",
              "142564                 1\n",
              "\n",
              "[142565 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 144
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LVot0F35f3WU"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "gb_test_predict.to_csv(\"gb_test_predict.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D2lgJxOJhB_8"
      },
      "source": [
        "##### AdaBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 464
        },
        "id": "sNwhu0mShEbN",
        "outputId": "82e6150f-5853-4674-e6c3-359ca8d4a7dd"
      },
      "source": [
        "DTC = DecisionTreeClassifier(max_depth = 1, random_state=42)\r\n",
        "ada_clf = AdaBoostClassifier(base_estimator = DTC, random_state=42) # default: estimators = 50, lr =1\r\n",
        "\r\n",
        "ada_params = {\r\n",
        "    'n_estimators' : [100,1000],\r\n",
        "    'learning_rate' : [0.01, 0.1, 1],\r\n",
        "}\r\n",
        "\r\n",
        "ada_grid_tst = GridSearchCV(ada_clf, param_grid = ada_params, cv = 3)\r\n",
        "ada_grid_tst.fit(train_X, train_y)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', ada_grid_tst.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(ada_grid_tst.best_score_))\r\n",
        "\r\n",
        "joblib.dump(ada_grid_tst,'ada_grid_tst.pkl')\r\n",
        "\r\n",
        "# predict\r\n",
        "ada_pred_tst = ada_grid_tst.best_estimator_.predict(test_X)\r\n",
        "\r\n",
        "ada_test_pred = pd.DataFrame(ada_pred_tst, columns =['AdaBoost'])\r\n",
        "ada_test_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'learning_rate': 1, 'n_estimators': 1000}\n",
            "최고 예측 정확도: 0.925487\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AdaBoost</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        AdaBoost\n",
              "0              0\n",
              "1              0\n",
              "2              0\n",
              "3              0\n",
              "4              0\n",
              "...          ...\n",
              "142560         1\n",
              "142561         1\n",
              "142562         1\n",
              "142563         1\n",
              "142564         1\n",
              "\n",
              "[142565 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 146
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cVx3NqfXhw0m"
      },
      "source": [
        "\r\n",
        "# submit\r\n",
        "ada_test_pred.to_csv(\"ada_test_pred.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yvfhyUwNiRMP"
      },
      "source": [
        "##### catBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kgLup61SiRbI",
        "outputId": "5d007817-0437-41d4-8f69-e5256db487c1"
      },
      "source": [
        "cb_clf_tst = CatBoostClassifier(random_state=42)\r\n",
        "cb_clf_tst.fit(train_X, train_y)\r\n",
        "\r\n",
        "joblib.dump(cb_clf_tst,'cb_clf_tst.pkl')\r\n",
        "\r\n",
        "cb_pred_tst = cb_clf_tst.predict(test_X)\r\n",
        "cb_test_pred = pd.DataFrame(cb_pred_tst, columns =['catboost'])\r\n",
        "cb_test_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Learning rate set to 0.079193\n",
            "0:\tlearn: 0.6147790\ttotal: 103ms\tremaining: 1m 42s\n",
            "1:\tlearn: 0.5534084\ttotal: 156ms\tremaining: 1m 17s\n",
            "2:\tlearn: 0.4877203\ttotal: 218ms\tremaining: 1m 12s\n",
            "3:\tlearn: 0.4497480\ttotal: 271ms\tremaining: 1m 7s\n",
            "4:\tlearn: 0.4042778\ttotal: 323ms\tremaining: 1m 4s\n",
            "5:\tlearn: 0.3760851\ttotal: 377ms\tremaining: 1m 2s\n",
            "6:\tlearn: 0.3465363\ttotal: 440ms\tremaining: 1m 2s\n",
            "7:\tlearn: 0.3233369\ttotal: 489ms\tremaining: 1m\n",
            "8:\tlearn: 0.3036278\ttotal: 547ms\tremaining: 1m\n",
            "9:\tlearn: 0.2910098\ttotal: 599ms\tremaining: 59.3s\n",
            "10:\tlearn: 0.2760490\ttotal: 666ms\tremaining: 59.9s\n",
            "11:\tlearn: 0.2668490\ttotal: 720ms\tremaining: 59.3s\n",
            "12:\tlearn: 0.2574775\ttotal: 772ms\tremaining: 58.6s\n",
            "13:\tlearn: 0.2503372\ttotal: 826ms\tremaining: 58.2s\n",
            "14:\tlearn: 0.2437103\ttotal: 883ms\tremaining: 58s\n",
            "15:\tlearn: 0.2351776\ttotal: 936ms\tremaining: 57.5s\n",
            "16:\tlearn: 0.2287148\ttotal: 988ms\tremaining: 57.1s\n",
            "17:\tlearn: 0.2235575\ttotal: 1.04s\tremaining: 56.9s\n",
            "18:\tlearn: 0.2186865\ttotal: 1.1s\tremaining: 56.8s\n",
            "19:\tlearn: 0.2135241\ttotal: 1.16s\tremaining: 56.6s\n",
            "20:\tlearn: 0.2090057\ttotal: 1.21s\tremaining: 56.5s\n",
            "21:\tlearn: 0.2058749\ttotal: 1.26s\tremaining: 56.2s\n",
            "22:\tlearn: 0.2016345\ttotal: 1.32s\tremaining: 56.2s\n",
            "23:\tlearn: 0.1974462\ttotal: 1.38s\tremaining: 56s\n",
            "24:\tlearn: 0.1944959\ttotal: 1.44s\tremaining: 56.2s\n",
            "25:\tlearn: 0.1912347\ttotal: 1.49s\tremaining: 56s\n",
            "26:\tlearn: 0.1885947\ttotal: 1.55s\tremaining: 55.8s\n",
            "27:\tlearn: 0.1862134\ttotal: 1.6s\tremaining: 55.5s\n",
            "28:\tlearn: 0.1836097\ttotal: 1.65s\tremaining: 55.3s\n",
            "29:\tlearn: 0.1811035\ttotal: 1.71s\tremaining: 55.2s\n",
            "30:\tlearn: 0.1787502\ttotal: 1.77s\tremaining: 55.2s\n",
            "31:\tlearn: 0.1758508\ttotal: 1.82s\tremaining: 55.2s\n",
            "32:\tlearn: 0.1744501\ttotal: 1.88s\tremaining: 55s\n",
            "33:\tlearn: 0.1724827\ttotal: 1.93s\tremaining: 54.9s\n",
            "34:\tlearn: 0.1705125\ttotal: 1.99s\tremaining: 54.8s\n",
            "35:\tlearn: 0.1689163\ttotal: 2.04s\tremaining: 54.7s\n",
            "36:\tlearn: 0.1661374\ttotal: 2.1s\tremaining: 54.8s\n",
            "37:\tlearn: 0.1644791\ttotal: 2.16s\tremaining: 54.7s\n",
            "38:\tlearn: 0.1629335\ttotal: 2.22s\tremaining: 54.6s\n",
            "39:\tlearn: 0.1602814\ttotal: 2.27s\tremaining: 54.5s\n",
            "40:\tlearn: 0.1589823\ttotal: 2.35s\tremaining: 55s\n",
            "41:\tlearn: 0.1580958\ttotal: 2.42s\tremaining: 55.1s\n",
            "42:\tlearn: 0.1564214\ttotal: 2.48s\tremaining: 55.1s\n",
            "43:\tlearn: 0.1552995\ttotal: 2.53s\tremaining: 54.9s\n",
            "44:\tlearn: 0.1542681\ttotal: 2.58s\tremaining: 54.8s\n",
            "45:\tlearn: 0.1529865\ttotal: 2.63s\tremaining: 54.6s\n",
            "46:\tlearn: 0.1515738\ttotal: 2.69s\tremaining: 54.5s\n",
            "47:\tlearn: 0.1504947\ttotal: 2.74s\tremaining: 54.3s\n",
            "48:\tlearn: 0.1493613\ttotal: 2.79s\tremaining: 54.2s\n",
            "49:\tlearn: 0.1481704\ttotal: 2.85s\tremaining: 54.2s\n",
            "50:\tlearn: 0.1469352\ttotal: 2.91s\tremaining: 54.2s\n",
            "51:\tlearn: 0.1460924\ttotal: 2.96s\tremaining: 54.1s\n",
            "52:\tlearn: 0.1451697\ttotal: 3.02s\tremaining: 53.9s\n",
            "53:\tlearn: 0.1442061\ttotal: 3.07s\tremaining: 53.8s\n",
            "54:\tlearn: 0.1429272\ttotal: 3.13s\tremaining: 53.7s\n",
            "55:\tlearn: 0.1420250\ttotal: 3.18s\tremaining: 53.6s\n",
            "56:\tlearn: 0.1412639\ttotal: 3.23s\tremaining: 53.5s\n",
            "57:\tlearn: 0.1404490\ttotal: 3.29s\tremaining: 53.4s\n",
            "58:\tlearn: 0.1396613\ttotal: 3.34s\tremaining: 53.3s\n",
            "59:\tlearn: 0.1386234\ttotal: 3.4s\tremaining: 53.2s\n",
            "60:\tlearn: 0.1379742\ttotal: 3.46s\tremaining: 53.3s\n",
            "61:\tlearn: 0.1376653\ttotal: 3.52s\tremaining: 53.2s\n",
            "62:\tlearn: 0.1371045\ttotal: 3.57s\tremaining: 53.1s\n",
            "63:\tlearn: 0.1364407\ttotal: 3.63s\tremaining: 53s\n",
            "64:\tlearn: 0.1359087\ttotal: 3.68s\tremaining: 52.9s\n",
            "65:\tlearn: 0.1354315\ttotal: 3.73s\tremaining: 52.8s\n",
            "66:\tlearn: 0.1348266\ttotal: 3.79s\tremaining: 52.8s\n",
            "67:\tlearn: 0.1342995\ttotal: 3.84s\tremaining: 52.7s\n",
            "68:\tlearn: 0.1335460\ttotal: 3.9s\tremaining: 52.6s\n",
            "69:\tlearn: 0.1330093\ttotal: 3.94s\tremaining: 52.4s\n",
            "70:\tlearn: 0.1322797\ttotal: 4s\tremaining: 52.4s\n",
            "71:\tlearn: 0.1318821\ttotal: 4.06s\tremaining: 52.3s\n",
            "72:\tlearn: 0.1313534\ttotal: 4.11s\tremaining: 52.2s\n",
            "73:\tlearn: 0.1305550\ttotal: 4.17s\tremaining: 52.2s\n",
            "74:\tlearn: 0.1299534\ttotal: 4.23s\tremaining: 52.1s\n",
            "75:\tlearn: 0.1293523\ttotal: 4.28s\tremaining: 52s\n",
            "76:\tlearn: 0.1288014\ttotal: 4.33s\tremaining: 52s\n",
            "77:\tlearn: 0.1284149\ttotal: 4.39s\tremaining: 51.9s\n",
            "78:\tlearn: 0.1276242\ttotal: 4.46s\tremaining: 52s\n",
            "79:\tlearn: 0.1272622\ttotal: 4.51s\tremaining: 51.9s\n",
            "80:\tlearn: 0.1264679\ttotal: 4.57s\tremaining: 51.8s\n",
            "81:\tlearn: 0.1262710\ttotal: 4.62s\tremaining: 51.7s\n",
            "82:\tlearn: 0.1257981\ttotal: 4.68s\tremaining: 51.7s\n",
            "83:\tlearn: 0.1254893\ttotal: 4.73s\tremaining: 51.6s\n",
            "84:\tlearn: 0.1251491\ttotal: 4.78s\tremaining: 51.5s\n",
            "85:\tlearn: 0.1248096\ttotal: 4.84s\tremaining: 51.4s\n",
            "86:\tlearn: 0.1243632\ttotal: 4.89s\tremaining: 51.4s\n",
            "87:\tlearn: 0.1239427\ttotal: 4.95s\tremaining: 51.3s\n",
            "88:\tlearn: 0.1236404\ttotal: 5s\tremaining: 51.2s\n",
            "89:\tlearn: 0.1233288\ttotal: 5.05s\tremaining: 51.1s\n",
            "90:\tlearn: 0.1229269\ttotal: 5.11s\tremaining: 51s\n",
            "91:\tlearn: 0.1224893\ttotal: 5.17s\tremaining: 51s\n",
            "92:\tlearn: 0.1219712\ttotal: 5.22s\tremaining: 50.9s\n",
            "93:\tlearn: 0.1215800\ttotal: 5.27s\tremaining: 50.8s\n",
            "94:\tlearn: 0.1212941\ttotal: 5.33s\tremaining: 50.8s\n",
            "95:\tlearn: 0.1209561\ttotal: 5.38s\tremaining: 50.6s\n",
            "96:\tlearn: 0.1204617\ttotal: 5.43s\tremaining: 50.6s\n",
            "97:\tlearn: 0.1200040\ttotal: 5.5s\tremaining: 50.6s\n",
            "98:\tlearn: 0.1198100\ttotal: 5.55s\tremaining: 50.6s\n",
            "99:\tlearn: 0.1193160\ttotal: 5.61s\tremaining: 50.5s\n",
            "100:\tlearn: 0.1189367\ttotal: 5.66s\tremaining: 50.4s\n",
            "101:\tlearn: 0.1186349\ttotal: 5.71s\tremaining: 50.3s\n",
            "102:\tlearn: 0.1182148\ttotal: 5.77s\tremaining: 50.3s\n",
            "103:\tlearn: 0.1178999\ttotal: 5.82s\tremaining: 50.2s\n",
            "104:\tlearn: 0.1174863\ttotal: 5.88s\tremaining: 50.1s\n",
            "105:\tlearn: 0.1171098\ttotal: 5.93s\tremaining: 50s\n",
            "106:\tlearn: 0.1166911\ttotal: 5.98s\tremaining: 49.9s\n",
            "107:\tlearn: 0.1164682\ttotal: 6.03s\tremaining: 49.8s\n",
            "108:\tlearn: 0.1162306\ttotal: 6.09s\tremaining: 49.8s\n",
            "109:\tlearn: 0.1160811\ttotal: 6.14s\tremaining: 49.7s\n",
            "110:\tlearn: 0.1157629\ttotal: 6.2s\tremaining: 49.7s\n",
            "111:\tlearn: 0.1154065\ttotal: 6.26s\tremaining: 49.6s\n",
            "112:\tlearn: 0.1148853\ttotal: 6.31s\tremaining: 49.6s\n",
            "113:\tlearn: 0.1147495\ttotal: 6.37s\tremaining: 49.5s\n",
            "114:\tlearn: 0.1143104\ttotal: 6.42s\tremaining: 49.4s\n",
            "115:\tlearn: 0.1140174\ttotal: 6.48s\tremaining: 49.4s\n",
            "116:\tlearn: 0.1137722\ttotal: 6.53s\tremaining: 49.3s\n",
            "117:\tlearn: 0.1135422\ttotal: 6.58s\tremaining: 49.2s\n",
            "118:\tlearn: 0.1131854\ttotal: 6.64s\tremaining: 49.2s\n",
            "119:\tlearn: 0.1130515\ttotal: 6.69s\tremaining: 49.1s\n",
            "120:\tlearn: 0.1128034\ttotal: 6.75s\tremaining: 49s\n",
            "121:\tlearn: 0.1125728\ttotal: 6.8s\tremaining: 48.9s\n",
            "122:\tlearn: 0.1123360\ttotal: 6.85s\tremaining: 48.9s\n",
            "123:\tlearn: 0.1119192\ttotal: 6.91s\tremaining: 48.8s\n",
            "124:\tlearn: 0.1115505\ttotal: 6.97s\tremaining: 48.8s\n",
            "125:\tlearn: 0.1114264\ttotal: 7.02s\tremaining: 48.7s\n",
            "126:\tlearn: 0.1112712\ttotal: 7.07s\tremaining: 48.6s\n",
            "127:\tlearn: 0.1110556\ttotal: 7.13s\tremaining: 48.5s\n",
            "128:\tlearn: 0.1107644\ttotal: 7.18s\tremaining: 48.5s\n",
            "129:\tlearn: 0.1105632\ttotal: 7.24s\tremaining: 48.4s\n",
            "130:\tlearn: 0.1103516\ttotal: 7.3s\tremaining: 48.4s\n",
            "131:\tlearn: 0.1101157\ttotal: 7.35s\tremaining: 48.3s\n",
            "132:\tlearn: 0.1098626\ttotal: 7.4s\tremaining: 48.3s\n",
            "133:\tlearn: 0.1096691\ttotal: 7.45s\tremaining: 48.2s\n",
            "134:\tlearn: 0.1094853\ttotal: 7.52s\tremaining: 48.2s\n",
            "135:\tlearn: 0.1092749\ttotal: 7.58s\tremaining: 48.2s\n",
            "136:\tlearn: 0.1090527\ttotal: 7.64s\tremaining: 48.1s\n",
            "137:\tlearn: 0.1088886\ttotal: 7.69s\tremaining: 48s\n",
            "138:\tlearn: 0.1086388\ttotal: 7.75s\tremaining: 48s\n",
            "139:\tlearn: 0.1084097\ttotal: 7.8s\tremaining: 47.9s\n",
            "140:\tlearn: 0.1081344\ttotal: 7.86s\tremaining: 47.9s\n",
            "141:\tlearn: 0.1079108\ttotal: 7.91s\tremaining: 47.8s\n",
            "142:\tlearn: 0.1077075\ttotal: 7.97s\tremaining: 47.8s\n",
            "143:\tlearn: 0.1073706\ttotal: 8.02s\tremaining: 47.7s\n",
            "144:\tlearn: 0.1071693\ttotal: 8.07s\tremaining: 47.6s\n",
            "145:\tlearn: 0.1069849\ttotal: 8.12s\tremaining: 47.5s\n",
            "146:\tlearn: 0.1067829\ttotal: 8.18s\tremaining: 47.4s\n",
            "147:\tlearn: 0.1065830\ttotal: 8.23s\tremaining: 47.4s\n",
            "148:\tlearn: 0.1064084\ttotal: 8.29s\tremaining: 47.3s\n",
            "149:\tlearn: 0.1061111\ttotal: 8.34s\tremaining: 47.3s\n",
            "150:\tlearn: 0.1058169\ttotal: 8.4s\tremaining: 47.2s\n",
            "151:\tlearn: 0.1055924\ttotal: 8.45s\tremaining: 47.1s\n",
            "152:\tlearn: 0.1054852\ttotal: 8.5s\tremaining: 47.1s\n",
            "153:\tlearn: 0.1053468\ttotal: 8.56s\tremaining: 47s\n",
            "154:\tlearn: 0.1051184\ttotal: 8.61s\tremaining: 47s\n",
            "155:\tlearn: 0.1048276\ttotal: 8.66s\tremaining: 46.9s\n",
            "156:\tlearn: 0.1046432\ttotal: 8.72s\tremaining: 46.8s\n",
            "157:\tlearn: 0.1044715\ttotal: 8.77s\tremaining: 46.7s\n",
            "158:\tlearn: 0.1041524\ttotal: 8.83s\tremaining: 46.7s\n",
            "159:\tlearn: 0.1037171\ttotal: 8.88s\tremaining: 46.6s\n",
            "160:\tlearn: 0.1035322\ttotal: 8.93s\tremaining: 46.6s\n",
            "161:\tlearn: 0.1033184\ttotal: 8.99s\tremaining: 46.5s\n",
            "162:\tlearn: 0.1031506\ttotal: 9.04s\tremaining: 46.4s\n",
            "163:\tlearn: 0.1029570\ttotal: 9.09s\tremaining: 46.4s\n",
            "164:\tlearn: 0.1027378\ttotal: 9.15s\tremaining: 46.3s\n",
            "165:\tlearn: 0.1024210\ttotal: 9.21s\tremaining: 46.3s\n",
            "166:\tlearn: 0.1022801\ttotal: 9.27s\tremaining: 46.3s\n",
            "167:\tlearn: 0.1020766\ttotal: 9.33s\tremaining: 46.2s\n",
            "168:\tlearn: 0.1018033\ttotal: 9.38s\tremaining: 46.1s\n",
            "169:\tlearn: 0.1015355\ttotal: 9.43s\tremaining: 46.1s\n",
            "170:\tlearn: 0.1013090\ttotal: 9.5s\tremaining: 46.1s\n",
            "171:\tlearn: 0.1009319\ttotal: 9.56s\tremaining: 46s\n",
            "172:\tlearn: 0.1007520\ttotal: 9.61s\tremaining: 45.9s\n",
            "173:\tlearn: 0.1005454\ttotal: 9.66s\tremaining: 45.8s\n",
            "174:\tlearn: 0.1003742\ttotal: 9.71s\tremaining: 45.8s\n",
            "175:\tlearn: 0.1000209\ttotal: 9.77s\tremaining: 45.7s\n",
            "176:\tlearn: 0.0998702\ttotal: 9.82s\tremaining: 45.6s\n",
            "177:\tlearn: 0.0996553\ttotal: 9.87s\tremaining: 45.6s\n",
            "178:\tlearn: 0.0994718\ttotal: 9.93s\tremaining: 45.5s\n",
            "179:\tlearn: 0.0992927\ttotal: 9.98s\tremaining: 45.5s\n",
            "180:\tlearn: 0.0991145\ttotal: 10s\tremaining: 45.4s\n",
            "181:\tlearn: 0.0989756\ttotal: 10.1s\tremaining: 45.3s\n",
            "182:\tlearn: 0.0987739\ttotal: 10.1s\tremaining: 45.2s\n",
            "183:\tlearn: 0.0985589\ttotal: 10.2s\tremaining: 45.2s\n",
            "184:\tlearn: 0.0983214\ttotal: 10.2s\tremaining: 45.1s\n",
            "185:\tlearn: 0.0981413\ttotal: 10.3s\tremaining: 45s\n",
            "186:\tlearn: 0.0978226\ttotal: 10.3s\tremaining: 45s\n",
            "187:\tlearn: 0.0976345\ttotal: 10.4s\tremaining: 44.9s\n",
            "188:\tlearn: 0.0973991\ttotal: 10.4s\tremaining: 44.8s\n",
            "189:\tlearn: 0.0972301\ttotal: 10.5s\tremaining: 44.8s\n",
            "190:\tlearn: 0.0970571\ttotal: 10.6s\tremaining: 44.7s\n",
            "191:\tlearn: 0.0967410\ttotal: 10.6s\tremaining: 44.7s\n",
            "192:\tlearn: 0.0964729\ttotal: 10.7s\tremaining: 44.6s\n",
            "193:\tlearn: 0.0963613\ttotal: 10.7s\tremaining: 44.5s\n",
            "194:\tlearn: 0.0962169\ttotal: 10.8s\tremaining: 44.5s\n",
            "195:\tlearn: 0.0960876\ttotal: 10.8s\tremaining: 44.4s\n",
            "196:\tlearn: 0.0959099\ttotal: 10.9s\tremaining: 44.4s\n",
            "197:\tlearn: 0.0957856\ttotal: 10.9s\tremaining: 44.3s\n",
            "198:\tlearn: 0.0955955\ttotal: 11s\tremaining: 44.2s\n",
            "199:\tlearn: 0.0954085\ttotal: 11s\tremaining: 44.2s\n",
            "200:\tlearn: 0.0952482\ttotal: 11.1s\tremaining: 44.1s\n",
            "201:\tlearn: 0.0950803\ttotal: 11.2s\tremaining: 44.1s\n",
            "202:\tlearn: 0.0949388\ttotal: 11.2s\tremaining: 44s\n",
            "203:\tlearn: 0.0946722\ttotal: 11.3s\tremaining: 44s\n",
            "204:\tlearn: 0.0944988\ttotal: 11.3s\tremaining: 43.9s\n",
            "205:\tlearn: 0.0943959\ttotal: 11.4s\tremaining: 43.8s\n",
            "206:\tlearn: 0.0942067\ttotal: 11.4s\tremaining: 43.7s\n",
            "207:\tlearn: 0.0940106\ttotal: 11.5s\tremaining: 43.7s\n",
            "208:\tlearn: 0.0938804\ttotal: 11.5s\tremaining: 43.7s\n",
            "209:\tlearn: 0.0936815\ttotal: 11.6s\tremaining: 43.6s\n",
            "210:\tlearn: 0.0935030\ttotal: 11.6s\tremaining: 43.5s\n",
            "211:\tlearn: 0.0933730\ttotal: 11.7s\tremaining: 43.5s\n",
            "212:\tlearn: 0.0932475\ttotal: 11.8s\tremaining: 43.4s\n",
            "213:\tlearn: 0.0931618\ttotal: 11.8s\tremaining: 43.3s\n",
            "214:\tlearn: 0.0929649\ttotal: 11.9s\tremaining: 43.3s\n",
            "215:\tlearn: 0.0927733\ttotal: 11.9s\tremaining: 43.2s\n",
            "216:\tlearn: 0.0926368\ttotal: 12s\tremaining: 43.1s\n",
            "217:\tlearn: 0.0925059\ttotal: 12s\tremaining: 43.1s\n",
            "218:\tlearn: 0.0922784\ttotal: 12.1s\tremaining: 43s\n",
            "219:\tlearn: 0.0921810\ttotal: 12.1s\tremaining: 42.9s\n",
            "220:\tlearn: 0.0920515\ttotal: 12.2s\tremaining: 42.9s\n",
            "221:\tlearn: 0.0919271\ttotal: 12.2s\tremaining: 42.8s\n",
            "222:\tlearn: 0.0917984\ttotal: 12.3s\tremaining: 42.7s\n",
            "223:\tlearn: 0.0916424\ttotal: 12.3s\tremaining: 42.7s\n",
            "224:\tlearn: 0.0914481\ttotal: 12.4s\tremaining: 42.6s\n",
            "225:\tlearn: 0.0912323\ttotal: 12.4s\tremaining: 42.6s\n",
            "226:\tlearn: 0.0911114\ttotal: 12.5s\tremaining: 42.5s\n",
            "227:\tlearn: 0.0909739\ttotal: 12.5s\tremaining: 42.5s\n",
            "228:\tlearn: 0.0908685\ttotal: 12.6s\tremaining: 42.4s\n",
            "229:\tlearn: 0.0906615\ttotal: 12.7s\tremaining: 42.4s\n",
            "230:\tlearn: 0.0905393\ttotal: 12.7s\tremaining: 42.4s\n",
            "231:\tlearn: 0.0903982\ttotal: 12.8s\tremaining: 42.3s\n",
            "232:\tlearn: 0.0903315\ttotal: 12.8s\tremaining: 42.3s\n",
            "233:\tlearn: 0.0902022\ttotal: 12.9s\tremaining: 42.2s\n",
            "234:\tlearn: 0.0900845\ttotal: 12.9s\tremaining: 42.2s\n",
            "235:\tlearn: 0.0899366\ttotal: 13s\tremaining: 42.1s\n",
            "236:\tlearn: 0.0897960\ttotal: 13.1s\tremaining: 42s\n",
            "237:\tlearn: 0.0896805\ttotal: 13.1s\tremaining: 42s\n",
            "238:\tlearn: 0.0895806\ttotal: 13.2s\tremaining: 41.9s\n",
            "239:\tlearn: 0.0894092\ttotal: 13.2s\tremaining: 41.8s\n",
            "240:\tlearn: 0.0893009\ttotal: 13.3s\tremaining: 41.8s\n",
            "241:\tlearn: 0.0891824\ttotal: 13.3s\tremaining: 41.7s\n",
            "242:\tlearn: 0.0890298\ttotal: 13.4s\tremaining: 41.6s\n",
            "243:\tlearn: 0.0889507\ttotal: 13.4s\tremaining: 41.6s\n",
            "244:\tlearn: 0.0887769\ttotal: 13.5s\tremaining: 41.5s\n",
            "245:\tlearn: 0.0886135\ttotal: 13.5s\tremaining: 41.5s\n",
            "246:\tlearn: 0.0884528\ttotal: 13.6s\tremaining: 41.4s\n",
            "247:\tlearn: 0.0883378\ttotal: 13.6s\tremaining: 41.4s\n",
            "248:\tlearn: 0.0882365\ttotal: 13.7s\tremaining: 41.3s\n",
            "249:\tlearn: 0.0880824\ttotal: 13.7s\tremaining: 41.2s\n",
            "250:\tlearn: 0.0879153\ttotal: 13.8s\tremaining: 41.2s\n",
            "251:\tlearn: 0.0878085\ttotal: 13.9s\tremaining: 41.1s\n",
            "252:\tlearn: 0.0877359\ttotal: 13.9s\tremaining: 41s\n",
            "253:\tlearn: 0.0876640\ttotal: 13.9s\tremaining: 41s\n",
            "254:\tlearn: 0.0874765\ttotal: 14s\tremaining: 40.9s\n",
            "255:\tlearn: 0.0873806\ttotal: 14.1s\tremaining: 40.9s\n",
            "256:\tlearn: 0.0872433\ttotal: 14.1s\tremaining: 40.8s\n",
            "257:\tlearn: 0.0870065\ttotal: 14.2s\tremaining: 40.8s\n",
            "258:\tlearn: 0.0868376\ttotal: 14.2s\tremaining: 40.7s\n",
            "259:\tlearn: 0.0865766\ttotal: 14.3s\tremaining: 40.7s\n",
            "260:\tlearn: 0.0864610\ttotal: 14.3s\tremaining: 40.6s\n",
            "261:\tlearn: 0.0863682\ttotal: 14.4s\tremaining: 40.5s\n",
            "262:\tlearn: 0.0862842\ttotal: 14.5s\tremaining: 40.5s\n",
            "263:\tlearn: 0.0860372\ttotal: 14.5s\tremaining: 40.4s\n",
            "264:\tlearn: 0.0858577\ttotal: 14.6s\tremaining: 40.4s\n",
            "265:\tlearn: 0.0857600\ttotal: 14.6s\tremaining: 40.3s\n",
            "266:\tlearn: 0.0855554\ttotal: 14.7s\tremaining: 40.3s\n",
            "267:\tlearn: 0.0854408\ttotal: 14.7s\tremaining: 40.2s\n",
            "268:\tlearn: 0.0853657\ttotal: 14.8s\tremaining: 40.2s\n",
            "269:\tlearn: 0.0852854\ttotal: 14.8s\tremaining: 40.1s\n",
            "270:\tlearn: 0.0851471\ttotal: 14.9s\tremaining: 40s\n",
            "271:\tlearn: 0.0849361\ttotal: 14.9s\tremaining: 40s\n",
            "272:\tlearn: 0.0848358\ttotal: 15s\tremaining: 39.9s\n",
            "273:\tlearn: 0.0847504\ttotal: 15s\tremaining: 39.9s\n",
            "274:\tlearn: 0.0845972\ttotal: 15.1s\tremaining: 39.8s\n",
            "275:\tlearn: 0.0844869\ttotal: 15.1s\tremaining: 39.7s\n",
            "276:\tlearn: 0.0844030\ttotal: 15.2s\tremaining: 39.7s\n",
            "277:\tlearn: 0.0843182\ttotal: 15.2s\tremaining: 39.6s\n",
            "278:\tlearn: 0.0841270\ttotal: 15.3s\tremaining: 39.5s\n",
            "279:\tlearn: 0.0840583\ttotal: 15.4s\tremaining: 39.5s\n",
            "280:\tlearn: 0.0839450\ttotal: 15.4s\tremaining: 39.4s\n",
            "281:\tlearn: 0.0837892\ttotal: 15.5s\tremaining: 39.3s\n",
            "282:\tlearn: 0.0835832\ttotal: 15.5s\tremaining: 39.3s\n",
            "283:\tlearn: 0.0835553\ttotal: 15.6s\tremaining: 39.2s\n",
            "284:\tlearn: 0.0833431\ttotal: 15.6s\tremaining: 39.2s\n",
            "285:\tlearn: 0.0831839\ttotal: 15.7s\tremaining: 39.2s\n",
            "286:\tlearn: 0.0830242\ttotal: 15.7s\tremaining: 39.1s\n",
            "287:\tlearn: 0.0827890\ttotal: 15.8s\tremaining: 39.1s\n",
            "288:\tlearn: 0.0826179\ttotal: 15.9s\tremaining: 39s\n",
            "289:\tlearn: 0.0825418\ttotal: 15.9s\tremaining: 39s\n",
            "290:\tlearn: 0.0824352\ttotal: 16s\tremaining: 38.9s\n",
            "291:\tlearn: 0.0823280\ttotal: 16s\tremaining: 38.8s\n",
            "292:\tlearn: 0.0822721\ttotal: 16.1s\tremaining: 38.8s\n",
            "293:\tlearn: 0.0821898\ttotal: 16.1s\tremaining: 38.7s\n",
            "294:\tlearn: 0.0820333\ttotal: 16.2s\tremaining: 38.7s\n",
            "295:\tlearn: 0.0819038\ttotal: 16.3s\tremaining: 38.7s\n",
            "296:\tlearn: 0.0817842\ttotal: 16.3s\tremaining: 38.6s\n",
            "297:\tlearn: 0.0816697\ttotal: 16.4s\tremaining: 38.5s\n",
            "298:\tlearn: 0.0815346\ttotal: 16.4s\tremaining: 38.5s\n",
            "299:\tlearn: 0.0814466\ttotal: 16.5s\tremaining: 38.4s\n",
            "300:\tlearn: 0.0813061\ttotal: 16.5s\tremaining: 38.4s\n",
            "301:\tlearn: 0.0811692\ttotal: 16.6s\tremaining: 38.3s\n",
            "302:\tlearn: 0.0809892\ttotal: 16.6s\tremaining: 38.3s\n",
            "303:\tlearn: 0.0808621\ttotal: 16.7s\tremaining: 38.2s\n",
            "304:\tlearn: 0.0807214\ttotal: 16.7s\tremaining: 38.2s\n",
            "305:\tlearn: 0.0806660\ttotal: 16.8s\tremaining: 38.1s\n",
            "306:\tlearn: 0.0805812\ttotal: 16.9s\tremaining: 38s\n",
            "307:\tlearn: 0.0804629\ttotal: 16.9s\tremaining: 38s\n",
            "308:\tlearn: 0.0803252\ttotal: 17s\tremaining: 37.9s\n",
            "309:\tlearn: 0.0801822\ttotal: 17s\tremaining: 37.9s\n",
            "310:\tlearn: 0.0800794\ttotal: 17.1s\tremaining: 37.8s\n",
            "311:\tlearn: 0.0799410\ttotal: 17.1s\tremaining: 37.8s\n",
            "312:\tlearn: 0.0798464\ttotal: 17.2s\tremaining: 37.7s\n",
            "313:\tlearn: 0.0797047\ttotal: 17.2s\tremaining: 37.6s\n",
            "314:\tlearn: 0.0795766\ttotal: 17.3s\tremaining: 37.6s\n",
            "315:\tlearn: 0.0794079\ttotal: 17.3s\tremaining: 37.5s\n",
            "316:\tlearn: 0.0793017\ttotal: 17.4s\tremaining: 37.5s\n",
            "317:\tlearn: 0.0792018\ttotal: 17.4s\tremaining: 37.4s\n",
            "318:\tlearn: 0.0791265\ttotal: 17.5s\tremaining: 37.4s\n",
            "319:\tlearn: 0.0790406\ttotal: 17.6s\tremaining: 37.3s\n",
            "320:\tlearn: 0.0788730\ttotal: 17.6s\tremaining: 37.3s\n",
            "321:\tlearn: 0.0788164\ttotal: 17.7s\tremaining: 37.2s\n",
            "322:\tlearn: 0.0786280\ttotal: 17.7s\tremaining: 37.1s\n",
            "323:\tlearn: 0.0785664\ttotal: 17.8s\tremaining: 37.1s\n",
            "324:\tlearn: 0.0784718\ttotal: 17.8s\tremaining: 37s\n",
            "325:\tlearn: 0.0783344\ttotal: 17.9s\tremaining: 37s\n",
            "326:\tlearn: 0.0782278\ttotal: 17.9s\tremaining: 36.9s\n",
            "327:\tlearn: 0.0781641\ttotal: 18s\tremaining: 36.8s\n",
            "328:\tlearn: 0.0780023\ttotal: 18s\tremaining: 36.8s\n",
            "329:\tlearn: 0.0778917\ttotal: 18.1s\tremaining: 36.7s\n",
            "330:\tlearn: 0.0777783\ttotal: 18.1s\tremaining: 36.7s\n",
            "331:\tlearn: 0.0776974\ttotal: 18.2s\tremaining: 36.6s\n",
            "332:\tlearn: 0.0776154\ttotal: 18.3s\tremaining: 36.6s\n",
            "333:\tlearn: 0.0775102\ttotal: 18.3s\tremaining: 36.5s\n",
            "334:\tlearn: 0.0774094\ttotal: 18.4s\tremaining: 36.5s\n",
            "335:\tlearn: 0.0773057\ttotal: 18.4s\tremaining: 36.4s\n",
            "336:\tlearn: 0.0770942\ttotal: 18.5s\tremaining: 36.3s\n",
            "337:\tlearn: 0.0770189\ttotal: 18.5s\tremaining: 36.3s\n",
            "338:\tlearn: 0.0769626\ttotal: 18.6s\tremaining: 36.2s\n",
            "339:\tlearn: 0.0768625\ttotal: 18.6s\tremaining: 36.2s\n",
            "340:\tlearn: 0.0767505\ttotal: 18.7s\tremaining: 36.1s\n",
            "341:\tlearn: 0.0766377\ttotal: 18.7s\tremaining: 36.1s\n",
            "342:\tlearn: 0.0765665\ttotal: 18.8s\tremaining: 36s\n",
            "343:\tlearn: 0.0764411\ttotal: 18.8s\tremaining: 35.9s\n",
            "344:\tlearn: 0.0763956\ttotal: 18.9s\tremaining: 35.9s\n",
            "345:\tlearn: 0.0763184\ttotal: 19s\tremaining: 35.8s\n",
            "346:\tlearn: 0.0762358\ttotal: 19s\tremaining: 35.8s\n",
            "347:\tlearn: 0.0761207\ttotal: 19.1s\tremaining: 35.7s\n",
            "348:\tlearn: 0.0760305\ttotal: 19.1s\tremaining: 35.7s\n",
            "349:\tlearn: 0.0759322\ttotal: 19.2s\tremaining: 35.6s\n",
            "350:\tlearn: 0.0758404\ttotal: 19.2s\tremaining: 35.6s\n",
            "351:\tlearn: 0.0757493\ttotal: 19.3s\tremaining: 35.5s\n",
            "352:\tlearn: 0.0756729\ttotal: 19.3s\tremaining: 35.4s\n",
            "353:\tlearn: 0.0756053\ttotal: 19.4s\tremaining: 35.4s\n",
            "354:\tlearn: 0.0755633\ttotal: 19.4s\tremaining: 35.3s\n",
            "355:\tlearn: 0.0754875\ttotal: 19.5s\tremaining: 35.2s\n",
            "356:\tlearn: 0.0754240\ttotal: 19.5s\tremaining: 35.2s\n",
            "357:\tlearn: 0.0753333\ttotal: 19.6s\tremaining: 35.1s\n",
            "358:\tlearn: 0.0752688\ttotal: 19.7s\tremaining: 35.1s\n",
            "359:\tlearn: 0.0752043\ttotal: 19.7s\tremaining: 35s\n",
            "360:\tlearn: 0.0751093\ttotal: 19.8s\tremaining: 35s\n",
            "361:\tlearn: 0.0750361\ttotal: 19.8s\tremaining: 34.9s\n",
            "362:\tlearn: 0.0749088\ttotal: 19.9s\tremaining: 34.9s\n",
            "363:\tlearn: 0.0748350\ttotal: 19.9s\tremaining: 34.8s\n",
            "364:\tlearn: 0.0747327\ttotal: 20s\tremaining: 34.7s\n",
            "365:\tlearn: 0.0746366\ttotal: 20s\tremaining: 34.7s\n",
            "366:\tlearn: 0.0745824\ttotal: 20.1s\tremaining: 34.6s\n",
            "367:\tlearn: 0.0744698\ttotal: 20.1s\tremaining: 34.6s\n",
            "368:\tlearn: 0.0743627\ttotal: 20.2s\tremaining: 34.5s\n",
            "369:\tlearn: 0.0743008\ttotal: 20.2s\tremaining: 34.5s\n",
            "370:\tlearn: 0.0742269\ttotal: 20.3s\tremaining: 34.4s\n",
            "371:\tlearn: 0.0741079\ttotal: 20.3s\tremaining: 34.3s\n",
            "372:\tlearn: 0.0740167\ttotal: 20.4s\tremaining: 34.3s\n",
            "373:\tlearn: 0.0738915\ttotal: 20.4s\tremaining: 34.2s\n",
            "374:\tlearn: 0.0737667\ttotal: 20.5s\tremaining: 34.2s\n",
            "375:\tlearn: 0.0737099\ttotal: 20.6s\tremaining: 34.1s\n",
            "376:\tlearn: 0.0736156\ttotal: 20.6s\tremaining: 34.1s\n",
            "377:\tlearn: 0.0735063\ttotal: 20.7s\tremaining: 34s\n",
            "378:\tlearn: 0.0734220\ttotal: 20.7s\tremaining: 34s\n",
            "379:\tlearn: 0.0733455\ttotal: 20.8s\tremaining: 33.9s\n",
            "380:\tlearn: 0.0731993\ttotal: 20.8s\tremaining: 33.8s\n",
            "381:\tlearn: 0.0730629\ttotal: 20.9s\tremaining: 33.8s\n",
            "382:\tlearn: 0.0729594\ttotal: 20.9s\tremaining: 33.7s\n",
            "383:\tlearn: 0.0728739\ttotal: 21s\tremaining: 33.7s\n",
            "384:\tlearn: 0.0727520\ttotal: 21s\tremaining: 33.6s\n",
            "385:\tlearn: 0.0726466\ttotal: 21.1s\tremaining: 33.6s\n",
            "386:\tlearn: 0.0725848\ttotal: 21.2s\tremaining: 33.5s\n",
            "387:\tlearn: 0.0724804\ttotal: 21.2s\tremaining: 33.5s\n",
            "388:\tlearn: 0.0723878\ttotal: 21.3s\tremaining: 33.4s\n",
            "389:\tlearn: 0.0722825\ttotal: 21.3s\tremaining: 33.3s\n",
            "390:\tlearn: 0.0721914\ttotal: 21.4s\tremaining: 33.3s\n",
            "391:\tlearn: 0.0721257\ttotal: 21.4s\tremaining: 33.2s\n",
            "392:\tlearn: 0.0720234\ttotal: 21.5s\tremaining: 33.2s\n",
            "393:\tlearn: 0.0718754\ttotal: 21.5s\tremaining: 33.1s\n",
            "394:\tlearn: 0.0717933\ttotal: 21.6s\tremaining: 33.1s\n",
            "395:\tlearn: 0.0717348\ttotal: 21.6s\tremaining: 33s\n",
            "396:\tlearn: 0.0716055\ttotal: 21.7s\tremaining: 33s\n",
            "397:\tlearn: 0.0715626\ttotal: 21.7s\tremaining: 32.9s\n",
            "398:\tlearn: 0.0715018\ttotal: 21.8s\tremaining: 32.8s\n",
            "399:\tlearn: 0.0713945\ttotal: 21.9s\tremaining: 32.8s\n",
            "400:\tlearn: 0.0712926\ttotal: 21.9s\tremaining: 32.7s\n",
            "401:\tlearn: 0.0712411\ttotal: 22s\tremaining: 32.7s\n",
            "402:\tlearn: 0.0711716\ttotal: 22s\tremaining: 32.6s\n",
            "403:\tlearn: 0.0710766\ttotal: 22.1s\tremaining: 32.6s\n",
            "404:\tlearn: 0.0709750\ttotal: 22.1s\tremaining: 32.5s\n",
            "405:\tlearn: 0.0708538\ttotal: 22.2s\tremaining: 32.5s\n",
            "406:\tlearn: 0.0707594\ttotal: 22.2s\tremaining: 32.4s\n",
            "407:\tlearn: 0.0706250\ttotal: 22.3s\tremaining: 32.3s\n",
            "408:\tlearn: 0.0705047\ttotal: 22.3s\tremaining: 32.3s\n",
            "409:\tlearn: 0.0704371\ttotal: 22.4s\tremaining: 32.2s\n",
            "410:\tlearn: 0.0703670\ttotal: 22.5s\tremaining: 32.2s\n",
            "411:\tlearn: 0.0702937\ttotal: 22.5s\tremaining: 32.1s\n",
            "412:\tlearn: 0.0702294\ttotal: 22.6s\tremaining: 32.1s\n",
            "413:\tlearn: 0.0701520\ttotal: 22.6s\tremaining: 32s\n",
            "414:\tlearn: 0.0700556\ttotal: 22.7s\tremaining: 32s\n",
            "415:\tlearn: 0.0699879\ttotal: 22.7s\tremaining: 31.9s\n",
            "416:\tlearn: 0.0698672\ttotal: 22.8s\tremaining: 31.8s\n",
            "417:\tlearn: 0.0697921\ttotal: 22.8s\tremaining: 31.8s\n",
            "418:\tlearn: 0.0696881\ttotal: 22.9s\tremaining: 31.7s\n",
            "419:\tlearn: 0.0696354\ttotal: 22.9s\tremaining: 31.7s\n",
            "420:\tlearn: 0.0695250\ttotal: 23s\tremaining: 31.6s\n",
            "421:\tlearn: 0.0694295\ttotal: 23.1s\tremaining: 31.6s\n",
            "422:\tlearn: 0.0693852\ttotal: 23.1s\tremaining: 31.5s\n",
            "423:\tlearn: 0.0692666\ttotal: 23.2s\tremaining: 31.5s\n",
            "424:\tlearn: 0.0691635\ttotal: 23.2s\tremaining: 31.4s\n",
            "425:\tlearn: 0.0691327\ttotal: 23.3s\tremaining: 31.4s\n",
            "426:\tlearn: 0.0690350\ttotal: 23.4s\tremaining: 31.3s\n",
            "427:\tlearn: 0.0689421\ttotal: 23.4s\tremaining: 31.3s\n",
            "428:\tlearn: 0.0688501\ttotal: 23.5s\tremaining: 31.2s\n",
            "429:\tlearn: 0.0688020\ttotal: 23.5s\tremaining: 31.2s\n",
            "430:\tlearn: 0.0687064\ttotal: 23.6s\tremaining: 31.1s\n",
            "431:\tlearn: 0.0686233\ttotal: 23.6s\tremaining: 31.1s\n",
            "432:\tlearn: 0.0685339\ttotal: 23.7s\tremaining: 31s\n",
            "433:\tlearn: 0.0684582\ttotal: 23.7s\tremaining: 31s\n",
            "434:\tlearn: 0.0684162\ttotal: 23.8s\tremaining: 30.9s\n",
            "435:\tlearn: 0.0683642\ttotal: 23.8s\tremaining: 30.8s\n",
            "436:\tlearn: 0.0682774\ttotal: 23.9s\tremaining: 30.8s\n",
            "437:\tlearn: 0.0682115\ttotal: 23.9s\tremaining: 30.7s\n",
            "438:\tlearn: 0.0681364\ttotal: 24s\tremaining: 30.7s\n",
            "439:\tlearn: 0.0680410\ttotal: 24.1s\tremaining: 30.6s\n",
            "440:\tlearn: 0.0679229\ttotal: 24.1s\tremaining: 30.6s\n",
            "441:\tlearn: 0.0678359\ttotal: 24.2s\tremaining: 30.5s\n",
            "442:\tlearn: 0.0677942\ttotal: 24.2s\tremaining: 30.5s\n",
            "443:\tlearn: 0.0676869\ttotal: 24.3s\tremaining: 30.4s\n",
            "444:\tlearn: 0.0675378\ttotal: 24.3s\tremaining: 30.3s\n",
            "445:\tlearn: 0.0673930\ttotal: 24.4s\tremaining: 30.3s\n",
            "446:\tlearn: 0.0673339\ttotal: 24.4s\tremaining: 30.2s\n",
            "447:\tlearn: 0.0672842\ttotal: 24.5s\tremaining: 30.2s\n",
            "448:\tlearn: 0.0671261\ttotal: 24.5s\tremaining: 30.1s\n",
            "449:\tlearn: 0.0670742\ttotal: 24.6s\tremaining: 30.1s\n",
            "450:\tlearn: 0.0669981\ttotal: 24.7s\tremaining: 30s\n",
            "451:\tlearn: 0.0668949\ttotal: 24.7s\tremaining: 30s\n",
            "452:\tlearn: 0.0668426\ttotal: 24.8s\tremaining: 29.9s\n",
            "453:\tlearn: 0.0667803\ttotal: 24.8s\tremaining: 29.9s\n",
            "454:\tlearn: 0.0666948\ttotal: 24.9s\tremaining: 29.8s\n",
            "455:\tlearn: 0.0666199\ttotal: 24.9s\tremaining: 29.7s\n",
            "456:\tlearn: 0.0665493\ttotal: 25s\tremaining: 29.7s\n",
            "457:\tlearn: 0.0664950\ttotal: 25s\tremaining: 29.6s\n",
            "458:\tlearn: 0.0664091\ttotal: 25.1s\tremaining: 29.6s\n",
            "459:\tlearn: 0.0663247\ttotal: 25.1s\tremaining: 29.5s\n",
            "460:\tlearn: 0.0662409\ttotal: 25.2s\tremaining: 29.5s\n",
            "461:\tlearn: 0.0662040\ttotal: 25.3s\tremaining: 29.4s\n",
            "462:\tlearn: 0.0661764\ttotal: 25.3s\tremaining: 29.3s\n",
            "463:\tlearn: 0.0661239\ttotal: 25.4s\tremaining: 29.3s\n",
            "464:\tlearn: 0.0660664\ttotal: 25.4s\tremaining: 29.2s\n",
            "465:\tlearn: 0.0660117\ttotal: 25.5s\tremaining: 29.2s\n",
            "466:\tlearn: 0.0659741\ttotal: 25.5s\tremaining: 29.1s\n",
            "467:\tlearn: 0.0658668\ttotal: 25.6s\tremaining: 29s\n",
            "468:\tlearn: 0.0657990\ttotal: 25.6s\tremaining: 29s\n",
            "469:\tlearn: 0.0657201\ttotal: 25.7s\tremaining: 28.9s\n",
            "470:\tlearn: 0.0656713\ttotal: 25.7s\tremaining: 28.9s\n",
            "471:\tlearn: 0.0656237\ttotal: 25.8s\tremaining: 28.8s\n",
            "472:\tlearn: 0.0655694\ttotal: 25.8s\tremaining: 28.8s\n",
            "473:\tlearn: 0.0655142\ttotal: 25.9s\tremaining: 28.7s\n",
            "474:\tlearn: 0.0654487\ttotal: 25.9s\tremaining: 28.7s\n",
            "475:\tlearn: 0.0653922\ttotal: 26s\tremaining: 28.6s\n",
            "476:\tlearn: 0.0653327\ttotal: 26s\tremaining: 28.5s\n",
            "477:\tlearn: 0.0652331\ttotal: 26.1s\tremaining: 28.5s\n",
            "478:\tlearn: 0.0651686\ttotal: 26.1s\tremaining: 28.4s\n",
            "479:\tlearn: 0.0651091\ttotal: 26.2s\tremaining: 28.4s\n",
            "480:\tlearn: 0.0650186\ttotal: 26.3s\tremaining: 28.3s\n",
            "481:\tlearn: 0.0649693\ttotal: 26.3s\tremaining: 28.3s\n",
            "482:\tlearn: 0.0649339\ttotal: 26.4s\tremaining: 28.2s\n",
            "483:\tlearn: 0.0649097\ttotal: 26.4s\tremaining: 28.2s\n",
            "484:\tlearn: 0.0648388\ttotal: 26.5s\tremaining: 28.1s\n",
            "485:\tlearn: 0.0647837\ttotal: 26.5s\tremaining: 28s\n",
            "486:\tlearn: 0.0647197\ttotal: 26.6s\tremaining: 28s\n",
            "487:\tlearn: 0.0646471\ttotal: 26.6s\tremaining: 27.9s\n",
            "488:\tlearn: 0.0645752\ttotal: 26.7s\tremaining: 27.9s\n",
            "489:\tlearn: 0.0645439\ttotal: 26.7s\tremaining: 27.8s\n",
            "490:\tlearn: 0.0645068\ttotal: 26.8s\tremaining: 27.8s\n",
            "491:\tlearn: 0.0644383\ttotal: 26.8s\tremaining: 27.7s\n",
            "492:\tlearn: 0.0644071\ttotal: 26.9s\tremaining: 27.7s\n",
            "493:\tlearn: 0.0643435\ttotal: 26.9s\tremaining: 27.6s\n",
            "494:\tlearn: 0.0642847\ttotal: 27s\tremaining: 27.5s\n",
            "495:\tlearn: 0.0642212\ttotal: 27.1s\tremaining: 27.5s\n",
            "496:\tlearn: 0.0641628\ttotal: 27.1s\tremaining: 27.4s\n",
            "497:\tlearn: 0.0641132\ttotal: 27.2s\tremaining: 27.4s\n",
            "498:\tlearn: 0.0640655\ttotal: 27.2s\tremaining: 27.3s\n",
            "499:\tlearn: 0.0640035\ttotal: 27.3s\tremaining: 27.3s\n",
            "500:\tlearn: 0.0639364\ttotal: 27.3s\tremaining: 27.2s\n",
            "501:\tlearn: 0.0638816\ttotal: 27.4s\tremaining: 27.2s\n",
            "502:\tlearn: 0.0638131\ttotal: 27.4s\tremaining: 27.1s\n",
            "503:\tlearn: 0.0637305\ttotal: 27.5s\tremaining: 27.1s\n",
            "504:\tlearn: 0.0636700\ttotal: 27.6s\tremaining: 27s\n",
            "505:\tlearn: 0.0636248\ttotal: 27.6s\tremaining: 26.9s\n",
            "506:\tlearn: 0.0635730\ttotal: 27.7s\tremaining: 26.9s\n",
            "507:\tlearn: 0.0634434\ttotal: 27.7s\tremaining: 26.8s\n",
            "508:\tlearn: 0.0633862\ttotal: 27.8s\tremaining: 26.8s\n",
            "509:\tlearn: 0.0633024\ttotal: 27.8s\tremaining: 26.7s\n",
            "510:\tlearn: 0.0632079\ttotal: 27.9s\tremaining: 26.7s\n",
            "511:\tlearn: 0.0631420\ttotal: 27.9s\tremaining: 26.6s\n",
            "512:\tlearn: 0.0630655\ttotal: 28s\tremaining: 26.6s\n",
            "513:\tlearn: 0.0630040\ttotal: 28s\tremaining: 26.5s\n",
            "514:\tlearn: 0.0629604\ttotal: 28.1s\tremaining: 26.5s\n",
            "515:\tlearn: 0.0628997\ttotal: 28.2s\tremaining: 26.4s\n",
            "516:\tlearn: 0.0628360\ttotal: 28.2s\tremaining: 26.4s\n",
            "517:\tlearn: 0.0627895\ttotal: 28.3s\tremaining: 26.3s\n",
            "518:\tlearn: 0.0626980\ttotal: 28.3s\tremaining: 26.2s\n",
            "519:\tlearn: 0.0626444\ttotal: 28.4s\tremaining: 26.2s\n",
            "520:\tlearn: 0.0625508\ttotal: 28.4s\tremaining: 26.1s\n",
            "521:\tlearn: 0.0624879\ttotal: 28.5s\tremaining: 26.1s\n",
            "522:\tlearn: 0.0624112\ttotal: 28.5s\tremaining: 26s\n",
            "523:\tlearn: 0.0623758\ttotal: 28.6s\tremaining: 26s\n",
            "524:\tlearn: 0.0622984\ttotal: 28.6s\tremaining: 25.9s\n",
            "525:\tlearn: 0.0622210\ttotal: 28.7s\tremaining: 25.9s\n",
            "526:\tlearn: 0.0621556\ttotal: 28.8s\tremaining: 25.8s\n",
            "527:\tlearn: 0.0620992\ttotal: 28.8s\tremaining: 25.8s\n",
            "528:\tlearn: 0.0620322\ttotal: 28.9s\tremaining: 25.7s\n",
            "529:\tlearn: 0.0619683\ttotal: 28.9s\tremaining: 25.6s\n",
            "530:\tlearn: 0.0618934\ttotal: 29s\tremaining: 25.6s\n",
            "531:\tlearn: 0.0618569\ttotal: 29s\tremaining: 25.5s\n",
            "532:\tlearn: 0.0618219\ttotal: 29.1s\tremaining: 25.5s\n",
            "533:\tlearn: 0.0617570\ttotal: 29.1s\tremaining: 25.4s\n",
            "534:\tlearn: 0.0617106\ttotal: 29.2s\tremaining: 25.4s\n",
            "535:\tlearn: 0.0616590\ttotal: 29.2s\tremaining: 25.3s\n",
            "536:\tlearn: 0.0615913\ttotal: 29.3s\tremaining: 25.3s\n",
            "537:\tlearn: 0.0615250\ttotal: 29.4s\tremaining: 25.2s\n",
            "538:\tlearn: 0.0614914\ttotal: 29.4s\tremaining: 25.1s\n",
            "539:\tlearn: 0.0614006\ttotal: 29.5s\tremaining: 25.1s\n",
            "540:\tlearn: 0.0613382\ttotal: 29.5s\tremaining: 25s\n",
            "541:\tlearn: 0.0612515\ttotal: 29.6s\tremaining: 25s\n",
            "542:\tlearn: 0.0611800\ttotal: 29.6s\tremaining: 24.9s\n",
            "543:\tlearn: 0.0611197\ttotal: 29.7s\tremaining: 24.9s\n",
            "544:\tlearn: 0.0610903\ttotal: 29.7s\tremaining: 24.8s\n",
            "545:\tlearn: 0.0610183\ttotal: 29.8s\tremaining: 24.8s\n",
            "546:\tlearn: 0.0609949\ttotal: 29.8s\tremaining: 24.7s\n",
            "547:\tlearn: 0.0609234\ttotal: 29.9s\tremaining: 24.7s\n",
            "548:\tlearn: 0.0608704\ttotal: 30s\tremaining: 24.6s\n",
            "549:\tlearn: 0.0608247\ttotal: 30s\tremaining: 24.6s\n",
            "550:\tlearn: 0.0607623\ttotal: 30.1s\tremaining: 24.5s\n",
            "551:\tlearn: 0.0606990\ttotal: 30.1s\tremaining: 24.4s\n",
            "552:\tlearn: 0.0606599\ttotal: 30.2s\tremaining: 24.4s\n",
            "553:\tlearn: 0.0605766\ttotal: 30.2s\tremaining: 24.3s\n",
            "554:\tlearn: 0.0604818\ttotal: 30.3s\tremaining: 24.3s\n",
            "555:\tlearn: 0.0603862\ttotal: 30.3s\tremaining: 24.2s\n",
            "556:\tlearn: 0.0603699\ttotal: 30.4s\tremaining: 24.2s\n",
            "557:\tlearn: 0.0603379\ttotal: 30.4s\tremaining: 24.1s\n",
            "558:\tlearn: 0.0602649\ttotal: 30.5s\tremaining: 24.1s\n",
            "559:\tlearn: 0.0602103\ttotal: 30.6s\tremaining: 24s\n",
            "560:\tlearn: 0.0601678\ttotal: 30.6s\tremaining: 24s\n",
            "561:\tlearn: 0.0601203\ttotal: 30.7s\tremaining: 23.9s\n",
            "562:\tlearn: 0.0600031\ttotal: 30.7s\tremaining: 23.9s\n",
            "563:\tlearn: 0.0599645\ttotal: 30.8s\tremaining: 23.8s\n",
            "564:\tlearn: 0.0599030\ttotal: 30.8s\tremaining: 23.7s\n",
            "565:\tlearn: 0.0598319\ttotal: 30.9s\tremaining: 23.7s\n",
            "566:\tlearn: 0.0597912\ttotal: 30.9s\tremaining: 23.6s\n",
            "567:\tlearn: 0.0597400\ttotal: 31s\tremaining: 23.6s\n",
            "568:\tlearn: 0.0596870\ttotal: 31.1s\tremaining: 23.5s\n",
            "569:\tlearn: 0.0596282\ttotal: 31.1s\tremaining: 23.5s\n",
            "570:\tlearn: 0.0595750\ttotal: 31.2s\tremaining: 23.4s\n",
            "571:\tlearn: 0.0594933\ttotal: 31.2s\tremaining: 23.4s\n",
            "572:\tlearn: 0.0594185\ttotal: 31.3s\tremaining: 23.3s\n",
            "573:\tlearn: 0.0593630\ttotal: 31.3s\tremaining: 23.2s\n",
            "574:\tlearn: 0.0592895\ttotal: 31.4s\tremaining: 23.2s\n",
            "575:\tlearn: 0.0592268\ttotal: 31.4s\tremaining: 23.1s\n",
            "576:\tlearn: 0.0591505\ttotal: 31.5s\tremaining: 23.1s\n",
            "577:\tlearn: 0.0591064\ttotal: 31.5s\tremaining: 23s\n",
            "578:\tlearn: 0.0590670\ttotal: 31.6s\tremaining: 23s\n",
            "579:\tlearn: 0.0590016\ttotal: 31.6s\tremaining: 22.9s\n",
            "580:\tlearn: 0.0589669\ttotal: 31.7s\tremaining: 22.9s\n",
            "581:\tlearn: 0.0589053\ttotal: 31.8s\tremaining: 22.8s\n",
            "582:\tlearn: 0.0588716\ttotal: 31.8s\tremaining: 22.8s\n",
            "583:\tlearn: 0.0587871\ttotal: 31.9s\tremaining: 22.7s\n",
            "584:\tlearn: 0.0586740\ttotal: 31.9s\tremaining: 22.6s\n",
            "585:\tlearn: 0.0586240\ttotal: 32s\tremaining: 22.6s\n",
            "586:\tlearn: 0.0585790\ttotal: 32s\tremaining: 22.5s\n",
            "587:\tlearn: 0.0585246\ttotal: 32.1s\tremaining: 22.5s\n",
            "588:\tlearn: 0.0584696\ttotal: 32.1s\tremaining: 22.4s\n",
            "589:\tlearn: 0.0584103\ttotal: 32.2s\tremaining: 22.4s\n",
            "590:\tlearn: 0.0583373\ttotal: 32.2s\tremaining: 22.3s\n",
            "591:\tlearn: 0.0582823\ttotal: 32.3s\tremaining: 22.3s\n",
            "592:\tlearn: 0.0581985\ttotal: 32.4s\tremaining: 22.2s\n",
            "593:\tlearn: 0.0581321\ttotal: 32.4s\tremaining: 22.1s\n",
            "594:\tlearn: 0.0580898\ttotal: 32.5s\tremaining: 22.1s\n",
            "595:\tlearn: 0.0580341\ttotal: 32.5s\tremaining: 22s\n",
            "596:\tlearn: 0.0579673\ttotal: 32.6s\tremaining: 22s\n",
            "597:\tlearn: 0.0579006\ttotal: 32.6s\tremaining: 21.9s\n",
            "598:\tlearn: 0.0578362\ttotal: 32.7s\tremaining: 21.9s\n",
            "599:\tlearn: 0.0577622\ttotal: 32.7s\tremaining: 21.8s\n",
            "600:\tlearn: 0.0577295\ttotal: 32.8s\tremaining: 21.8s\n",
            "601:\tlearn: 0.0576582\ttotal: 32.8s\tremaining: 21.7s\n",
            "602:\tlearn: 0.0576206\ttotal: 32.9s\tremaining: 21.7s\n",
            "603:\tlearn: 0.0575833\ttotal: 32.9s\tremaining: 21.6s\n",
            "604:\tlearn: 0.0575397\ttotal: 33s\tremaining: 21.5s\n",
            "605:\tlearn: 0.0575028\ttotal: 33.1s\tremaining: 21.5s\n",
            "606:\tlearn: 0.0574829\ttotal: 33.1s\tremaining: 21.4s\n",
            "607:\tlearn: 0.0574058\ttotal: 33.2s\tremaining: 21.4s\n",
            "608:\tlearn: 0.0573474\ttotal: 33.2s\tremaining: 21.3s\n",
            "609:\tlearn: 0.0572872\ttotal: 33.3s\tremaining: 21.3s\n",
            "610:\tlearn: 0.0572375\ttotal: 33.3s\tremaining: 21.2s\n",
            "611:\tlearn: 0.0571637\ttotal: 33.4s\tremaining: 21.2s\n",
            "612:\tlearn: 0.0571150\ttotal: 33.5s\tremaining: 21.1s\n",
            "613:\tlearn: 0.0570624\ttotal: 33.5s\tremaining: 21.1s\n",
            "614:\tlearn: 0.0570215\ttotal: 33.6s\tremaining: 21s\n",
            "615:\tlearn: 0.0569619\ttotal: 33.6s\tremaining: 21s\n",
            "616:\tlearn: 0.0569052\ttotal: 33.7s\tremaining: 20.9s\n",
            "617:\tlearn: 0.0568746\ttotal: 33.8s\tremaining: 20.9s\n",
            "618:\tlearn: 0.0568255\ttotal: 33.8s\tremaining: 20.8s\n",
            "619:\tlearn: 0.0567568\ttotal: 33.9s\tremaining: 20.8s\n",
            "620:\tlearn: 0.0567272\ttotal: 33.9s\tremaining: 20.7s\n",
            "621:\tlearn: 0.0566413\ttotal: 34s\tremaining: 20.6s\n",
            "622:\tlearn: 0.0565734\ttotal: 34s\tremaining: 20.6s\n",
            "623:\tlearn: 0.0564696\ttotal: 34.1s\tremaining: 20.5s\n",
            "624:\tlearn: 0.0564085\ttotal: 34.1s\tremaining: 20.5s\n",
            "625:\tlearn: 0.0563394\ttotal: 34.2s\tremaining: 20.4s\n",
            "626:\tlearn: 0.0563208\ttotal: 34.2s\tremaining: 20.4s\n",
            "627:\tlearn: 0.0562622\ttotal: 34.3s\tremaining: 20.3s\n",
            "628:\tlearn: 0.0562008\ttotal: 34.3s\tremaining: 20.3s\n",
            "629:\tlearn: 0.0561566\ttotal: 34.4s\tremaining: 20.2s\n",
            "630:\tlearn: 0.0561355\ttotal: 34.5s\tremaining: 20.1s\n",
            "631:\tlearn: 0.0560871\ttotal: 34.5s\tremaining: 20.1s\n",
            "632:\tlearn: 0.0560408\ttotal: 34.6s\tremaining: 20s\n",
            "633:\tlearn: 0.0559781\ttotal: 34.6s\tremaining: 20s\n",
            "634:\tlearn: 0.0559155\ttotal: 34.7s\tremaining: 19.9s\n",
            "635:\tlearn: 0.0558312\ttotal: 34.7s\tremaining: 19.9s\n",
            "636:\tlearn: 0.0558049\ttotal: 34.8s\tremaining: 19.8s\n",
            "637:\tlearn: 0.0557474\ttotal: 34.8s\tremaining: 19.8s\n",
            "638:\tlearn: 0.0556860\ttotal: 34.9s\tremaining: 19.7s\n",
            "639:\tlearn: 0.0556297\ttotal: 35s\tremaining: 19.7s\n",
            "640:\tlearn: 0.0555995\ttotal: 35s\tremaining: 19.6s\n",
            "641:\tlearn: 0.0554958\ttotal: 35.1s\tremaining: 19.6s\n",
            "642:\tlearn: 0.0554536\ttotal: 35.1s\tremaining: 19.5s\n",
            "643:\tlearn: 0.0554088\ttotal: 35.2s\tremaining: 19.4s\n",
            "644:\tlearn: 0.0553679\ttotal: 35.2s\tremaining: 19.4s\n",
            "645:\tlearn: 0.0553110\ttotal: 35.3s\tremaining: 19.3s\n",
            "646:\tlearn: 0.0552716\ttotal: 35.3s\tremaining: 19.3s\n",
            "647:\tlearn: 0.0552004\ttotal: 35.4s\tremaining: 19.2s\n",
            "648:\tlearn: 0.0551521\ttotal: 35.4s\tremaining: 19.2s\n",
            "649:\tlearn: 0.0550850\ttotal: 35.5s\tremaining: 19.1s\n",
            "650:\tlearn: 0.0550351\ttotal: 35.5s\tremaining: 19.1s\n",
            "651:\tlearn: 0.0549982\ttotal: 35.6s\tremaining: 19s\n",
            "652:\tlearn: 0.0549598\ttotal: 35.7s\tremaining: 18.9s\n",
            "653:\tlearn: 0.0549141\ttotal: 35.7s\tremaining: 18.9s\n",
            "654:\tlearn: 0.0548680\ttotal: 35.8s\tremaining: 18.8s\n",
            "655:\tlearn: 0.0548209\ttotal: 35.8s\tremaining: 18.8s\n",
            "656:\tlearn: 0.0547530\ttotal: 35.9s\tremaining: 18.7s\n",
            "657:\tlearn: 0.0547084\ttotal: 35.9s\tremaining: 18.7s\n",
            "658:\tlearn: 0.0546612\ttotal: 36s\tremaining: 18.6s\n",
            "659:\tlearn: 0.0546055\ttotal: 36s\tremaining: 18.6s\n",
            "660:\tlearn: 0.0545717\ttotal: 36.1s\tremaining: 18.5s\n",
            "661:\tlearn: 0.0545456\ttotal: 36.1s\tremaining: 18.5s\n",
            "662:\tlearn: 0.0545179\ttotal: 36.2s\tremaining: 18.4s\n",
            "663:\tlearn: 0.0544521\ttotal: 36.2s\tremaining: 18.3s\n",
            "664:\tlearn: 0.0543934\ttotal: 36.3s\tremaining: 18.3s\n",
            "665:\tlearn: 0.0543108\ttotal: 36.4s\tremaining: 18.2s\n",
            "666:\tlearn: 0.0542853\ttotal: 36.4s\tremaining: 18.2s\n",
            "667:\tlearn: 0.0542120\ttotal: 36.5s\tremaining: 18.1s\n",
            "668:\tlearn: 0.0541698\ttotal: 36.5s\tremaining: 18.1s\n",
            "669:\tlearn: 0.0541107\ttotal: 36.6s\tremaining: 18s\n",
            "670:\tlearn: 0.0540408\ttotal: 36.6s\tremaining: 18s\n",
            "671:\tlearn: 0.0539882\ttotal: 36.7s\tremaining: 17.9s\n",
            "672:\tlearn: 0.0539261\ttotal: 36.8s\tremaining: 17.9s\n",
            "673:\tlearn: 0.0538785\ttotal: 36.8s\tremaining: 17.8s\n",
            "674:\tlearn: 0.0538230\ttotal: 36.9s\tremaining: 17.8s\n",
            "675:\tlearn: 0.0537631\ttotal: 36.9s\tremaining: 17.7s\n",
            "676:\tlearn: 0.0537142\ttotal: 37s\tremaining: 17.6s\n",
            "677:\tlearn: 0.0536886\ttotal: 37s\tremaining: 17.6s\n",
            "678:\tlearn: 0.0536499\ttotal: 37.1s\tremaining: 17.5s\n",
            "679:\tlearn: 0.0536157\ttotal: 37.1s\tremaining: 17.5s\n",
            "680:\tlearn: 0.0535688\ttotal: 37.2s\tremaining: 17.4s\n",
            "681:\tlearn: 0.0535240\ttotal: 37.3s\tremaining: 17.4s\n",
            "682:\tlearn: 0.0534857\ttotal: 37.3s\tremaining: 17.3s\n",
            "683:\tlearn: 0.0534479\ttotal: 37.4s\tremaining: 17.3s\n",
            "684:\tlearn: 0.0533994\ttotal: 37.4s\tremaining: 17.2s\n",
            "685:\tlearn: 0.0533514\ttotal: 37.5s\tremaining: 17.1s\n",
            "686:\tlearn: 0.0533239\ttotal: 37.5s\tremaining: 17.1s\n",
            "687:\tlearn: 0.0532456\ttotal: 37.6s\tremaining: 17s\n",
            "688:\tlearn: 0.0532038\ttotal: 37.6s\tremaining: 17s\n",
            "689:\tlearn: 0.0531659\ttotal: 37.7s\tremaining: 16.9s\n",
            "690:\tlearn: 0.0531270\ttotal: 37.7s\tremaining: 16.9s\n",
            "691:\tlearn: 0.0530782\ttotal: 37.8s\tremaining: 16.8s\n",
            "692:\tlearn: 0.0530265\ttotal: 37.9s\tremaining: 16.8s\n",
            "693:\tlearn: 0.0529754\ttotal: 37.9s\tremaining: 16.7s\n",
            "694:\tlearn: 0.0529492\ttotal: 38s\tremaining: 16.7s\n",
            "695:\tlearn: 0.0529087\ttotal: 38s\tremaining: 16.6s\n",
            "696:\tlearn: 0.0528465\ttotal: 38.1s\tremaining: 16.6s\n",
            "697:\tlearn: 0.0527991\ttotal: 38.1s\tremaining: 16.5s\n",
            "698:\tlearn: 0.0527645\ttotal: 38.2s\tremaining: 16.4s\n",
            "699:\tlearn: 0.0527254\ttotal: 38.2s\tremaining: 16.4s\n",
            "700:\tlearn: 0.0526835\ttotal: 38.3s\tremaining: 16.3s\n",
            "701:\tlearn: 0.0526091\ttotal: 38.3s\tremaining: 16.3s\n",
            "702:\tlearn: 0.0525766\ttotal: 38.4s\tremaining: 16.2s\n",
            "703:\tlearn: 0.0525186\ttotal: 38.4s\tremaining: 16.2s\n",
            "704:\tlearn: 0.0524808\ttotal: 38.5s\tremaining: 16.1s\n",
            "705:\tlearn: 0.0524449\ttotal: 38.6s\tremaining: 16.1s\n",
            "706:\tlearn: 0.0524129\ttotal: 38.6s\tremaining: 16s\n",
            "707:\tlearn: 0.0523868\ttotal: 38.7s\tremaining: 15.9s\n",
            "708:\tlearn: 0.0523089\ttotal: 38.7s\tremaining: 15.9s\n",
            "709:\tlearn: 0.0522561\ttotal: 38.8s\tremaining: 15.8s\n",
            "710:\tlearn: 0.0522267\ttotal: 38.8s\tremaining: 15.8s\n",
            "711:\tlearn: 0.0521743\ttotal: 38.9s\tremaining: 15.7s\n",
            "712:\tlearn: 0.0521080\ttotal: 38.9s\tremaining: 15.7s\n",
            "713:\tlearn: 0.0520413\ttotal: 39s\tremaining: 15.6s\n",
            "714:\tlearn: 0.0519818\ttotal: 39.1s\tremaining: 15.6s\n",
            "715:\tlearn: 0.0519537\ttotal: 39.1s\tremaining: 15.5s\n",
            "716:\tlearn: 0.0518910\ttotal: 39.2s\tremaining: 15.5s\n",
            "717:\tlearn: 0.0518340\ttotal: 39.2s\tremaining: 15.4s\n",
            "718:\tlearn: 0.0518005\ttotal: 39.3s\tremaining: 15.3s\n",
            "719:\tlearn: 0.0517905\ttotal: 39.3s\tremaining: 15.3s\n",
            "720:\tlearn: 0.0517413\ttotal: 39.4s\tremaining: 15.2s\n",
            "721:\tlearn: 0.0516841\ttotal: 39.4s\tremaining: 15.2s\n",
            "722:\tlearn: 0.0516339\ttotal: 39.5s\tremaining: 15.1s\n",
            "723:\tlearn: 0.0516174\ttotal: 39.5s\tremaining: 15.1s\n",
            "724:\tlearn: 0.0515646\ttotal: 39.6s\tremaining: 15s\n",
            "725:\tlearn: 0.0515179\ttotal: 39.6s\tremaining: 15s\n",
            "726:\tlearn: 0.0514737\ttotal: 39.7s\tremaining: 14.9s\n",
            "727:\tlearn: 0.0514205\ttotal: 39.7s\tremaining: 14.9s\n",
            "728:\tlearn: 0.0513929\ttotal: 39.8s\tremaining: 14.8s\n",
            "729:\tlearn: 0.0513703\ttotal: 39.9s\tremaining: 14.7s\n",
            "730:\tlearn: 0.0513223\ttotal: 39.9s\tremaining: 14.7s\n",
            "731:\tlearn: 0.0512766\ttotal: 40s\tremaining: 14.6s\n",
            "732:\tlearn: 0.0512268\ttotal: 40s\tremaining: 14.6s\n",
            "733:\tlearn: 0.0511873\ttotal: 40.1s\tremaining: 14.5s\n",
            "734:\tlearn: 0.0511440\ttotal: 40.1s\tremaining: 14.5s\n",
            "735:\tlearn: 0.0510987\ttotal: 40.2s\tremaining: 14.4s\n",
            "736:\tlearn: 0.0510577\ttotal: 40.2s\tremaining: 14.4s\n",
            "737:\tlearn: 0.0510231\ttotal: 40.3s\tremaining: 14.3s\n",
            "738:\tlearn: 0.0509981\ttotal: 40.3s\tremaining: 14.2s\n",
            "739:\tlearn: 0.0509516\ttotal: 40.4s\tremaining: 14.2s\n",
            "740:\tlearn: 0.0509279\ttotal: 40.5s\tremaining: 14.1s\n",
            "741:\tlearn: 0.0508676\ttotal: 40.5s\tremaining: 14.1s\n",
            "742:\tlearn: 0.0508284\ttotal: 40.6s\tremaining: 14s\n",
            "743:\tlearn: 0.0507532\ttotal: 40.6s\tremaining: 14s\n",
            "744:\tlearn: 0.0507233\ttotal: 40.7s\tremaining: 13.9s\n",
            "745:\tlearn: 0.0506958\ttotal: 40.8s\tremaining: 13.9s\n",
            "746:\tlearn: 0.0506440\ttotal: 40.8s\tremaining: 13.8s\n",
            "747:\tlearn: 0.0506111\ttotal: 40.9s\tremaining: 13.8s\n",
            "748:\tlearn: 0.0505920\ttotal: 40.9s\tremaining: 13.7s\n",
            "749:\tlearn: 0.0505234\ttotal: 41s\tremaining: 13.7s\n",
            "750:\tlearn: 0.0504756\ttotal: 41s\tremaining: 13.6s\n",
            "751:\tlearn: 0.0504345\ttotal: 41.1s\tremaining: 13.5s\n",
            "752:\tlearn: 0.0503828\ttotal: 41.1s\tremaining: 13.5s\n",
            "753:\tlearn: 0.0503483\ttotal: 41.2s\tremaining: 13.4s\n",
            "754:\tlearn: 0.0502880\ttotal: 41.2s\tremaining: 13.4s\n",
            "755:\tlearn: 0.0502617\ttotal: 41.3s\tremaining: 13.3s\n",
            "756:\tlearn: 0.0502147\ttotal: 41.4s\tremaining: 13.3s\n",
            "757:\tlearn: 0.0501964\ttotal: 41.4s\tremaining: 13.2s\n",
            "758:\tlearn: 0.0501217\ttotal: 41.5s\tremaining: 13.2s\n",
            "759:\tlearn: 0.0500726\ttotal: 41.5s\tremaining: 13.1s\n",
            "760:\tlearn: 0.0500568\ttotal: 41.6s\tremaining: 13.1s\n",
            "761:\tlearn: 0.0500279\ttotal: 41.6s\tremaining: 13s\n",
            "762:\tlearn: 0.0499743\ttotal: 41.7s\tremaining: 12.9s\n",
            "763:\tlearn: 0.0499344\ttotal: 41.7s\tremaining: 12.9s\n",
            "764:\tlearn: 0.0498892\ttotal: 41.8s\tremaining: 12.8s\n",
            "765:\tlearn: 0.0498796\ttotal: 41.8s\tremaining: 12.8s\n",
            "766:\tlearn: 0.0498281\ttotal: 41.9s\tremaining: 12.7s\n",
            "767:\tlearn: 0.0497991\ttotal: 41.9s\tremaining: 12.7s\n",
            "768:\tlearn: 0.0497520\ttotal: 42s\tremaining: 12.6s\n",
            "769:\tlearn: 0.0497036\ttotal: 42.1s\tremaining: 12.6s\n",
            "770:\tlearn: 0.0496773\ttotal: 42.1s\tremaining: 12.5s\n",
            "771:\tlearn: 0.0496596\ttotal: 42.2s\tremaining: 12.5s\n",
            "772:\tlearn: 0.0496181\ttotal: 42.2s\tremaining: 12.4s\n",
            "773:\tlearn: 0.0495749\ttotal: 42.3s\tremaining: 12.3s\n",
            "774:\tlearn: 0.0495533\ttotal: 42.3s\tremaining: 12.3s\n",
            "775:\tlearn: 0.0495064\ttotal: 42.4s\tremaining: 12.2s\n",
            "776:\tlearn: 0.0494818\ttotal: 42.4s\tremaining: 12.2s\n",
            "777:\tlearn: 0.0494382\ttotal: 42.5s\tremaining: 12.1s\n",
            "778:\tlearn: 0.0493932\ttotal: 42.5s\tremaining: 12.1s\n",
            "779:\tlearn: 0.0493607\ttotal: 42.6s\tremaining: 12s\n",
            "780:\tlearn: 0.0493210\ttotal: 42.7s\tremaining: 12s\n",
            "781:\tlearn: 0.0492987\ttotal: 42.7s\tremaining: 11.9s\n",
            "782:\tlearn: 0.0492456\ttotal: 42.8s\tremaining: 11.9s\n",
            "783:\tlearn: 0.0491977\ttotal: 42.8s\tremaining: 11.8s\n",
            "784:\tlearn: 0.0491560\ttotal: 42.9s\tremaining: 11.7s\n",
            "785:\tlearn: 0.0491118\ttotal: 43s\tremaining: 11.7s\n",
            "786:\tlearn: 0.0490518\ttotal: 43s\tremaining: 11.6s\n",
            "787:\tlearn: 0.0490279\ttotal: 43.1s\tremaining: 11.6s\n",
            "788:\tlearn: 0.0489475\ttotal: 43.1s\tremaining: 11.5s\n",
            "789:\tlearn: 0.0489051\ttotal: 43.2s\tremaining: 11.5s\n",
            "790:\tlearn: 0.0489003\ttotal: 43.2s\tremaining: 11.4s\n",
            "791:\tlearn: 0.0488608\ttotal: 43.3s\tremaining: 11.4s\n",
            "792:\tlearn: 0.0488342\ttotal: 43.3s\tremaining: 11.3s\n",
            "793:\tlearn: 0.0487946\ttotal: 43.4s\tremaining: 11.3s\n",
            "794:\tlearn: 0.0487471\ttotal: 43.5s\tremaining: 11.2s\n",
            "795:\tlearn: 0.0487128\ttotal: 43.5s\tremaining: 11.1s\n",
            "796:\tlearn: 0.0486868\ttotal: 43.6s\tremaining: 11.1s\n",
            "797:\tlearn: 0.0486701\ttotal: 43.6s\tremaining: 11s\n",
            "798:\tlearn: 0.0486351\ttotal: 43.7s\tremaining: 11s\n",
            "799:\tlearn: 0.0485782\ttotal: 43.7s\tremaining: 10.9s\n",
            "800:\tlearn: 0.0485075\ttotal: 43.8s\tremaining: 10.9s\n",
            "801:\tlearn: 0.0484666\ttotal: 43.9s\tremaining: 10.8s\n",
            "802:\tlearn: 0.0484137\ttotal: 43.9s\tremaining: 10.8s\n",
            "803:\tlearn: 0.0483773\ttotal: 44s\tremaining: 10.7s\n",
            "804:\tlearn: 0.0483376\ttotal: 44s\tremaining: 10.7s\n",
            "805:\tlearn: 0.0483069\ttotal: 44.1s\tremaining: 10.6s\n",
            "806:\tlearn: 0.0482617\ttotal: 44.1s\tremaining: 10.6s\n",
            "807:\tlearn: 0.0482259\ttotal: 44.2s\tremaining: 10.5s\n",
            "808:\tlearn: 0.0481894\ttotal: 44.2s\tremaining: 10.4s\n",
            "809:\tlearn: 0.0481630\ttotal: 44.3s\tremaining: 10.4s\n",
            "810:\tlearn: 0.0481231\ttotal: 44.4s\tremaining: 10.3s\n",
            "811:\tlearn: 0.0481018\ttotal: 44.4s\tremaining: 10.3s\n",
            "812:\tlearn: 0.0480739\ttotal: 44.4s\tremaining: 10.2s\n",
            "813:\tlearn: 0.0480316\ttotal: 44.5s\tremaining: 10.2s\n",
            "814:\tlearn: 0.0479979\ttotal: 44.6s\tremaining: 10.1s\n",
            "815:\tlearn: 0.0479592\ttotal: 44.6s\tremaining: 10.1s\n",
            "816:\tlearn: 0.0479154\ttotal: 44.7s\tremaining: 10s\n",
            "817:\tlearn: 0.0478861\ttotal: 44.7s\tremaining: 9.95s\n",
            "818:\tlearn: 0.0478571\ttotal: 44.8s\tremaining: 9.89s\n",
            "819:\tlearn: 0.0478367\ttotal: 44.8s\tremaining: 9.84s\n",
            "820:\tlearn: 0.0477995\ttotal: 44.9s\tremaining: 9.79s\n",
            "821:\tlearn: 0.0477794\ttotal: 44.9s\tremaining: 9.73s\n",
            "822:\tlearn: 0.0477495\ttotal: 45s\tremaining: 9.68s\n",
            "823:\tlearn: 0.0477349\ttotal: 45s\tremaining: 9.62s\n",
            "824:\tlearn: 0.0476936\ttotal: 45.1s\tremaining: 9.57s\n",
            "825:\tlearn: 0.0476679\ttotal: 45.2s\tremaining: 9.51s\n",
            "826:\tlearn: 0.0476238\ttotal: 45.2s\tremaining: 9.46s\n",
            "827:\tlearn: 0.0475946\ttotal: 45.3s\tremaining: 9.4s\n",
            "828:\tlearn: 0.0475648\ttotal: 45.3s\tremaining: 9.35s\n",
            "829:\tlearn: 0.0475141\ttotal: 45.4s\tremaining: 9.29s\n",
            "830:\tlearn: 0.0474678\ttotal: 45.4s\tremaining: 9.24s\n",
            "831:\tlearn: 0.0474473\ttotal: 45.5s\tremaining: 9.18s\n",
            "832:\tlearn: 0.0474196\ttotal: 45.5s\tremaining: 9.13s\n",
            "833:\tlearn: 0.0473726\ttotal: 45.6s\tremaining: 9.07s\n",
            "834:\tlearn: 0.0473667\ttotal: 45.6s\tremaining: 9.02s\n",
            "835:\tlearn: 0.0473226\ttotal: 45.7s\tremaining: 8.96s\n",
            "836:\tlearn: 0.0473058\ttotal: 45.7s\tremaining: 8.91s\n",
            "837:\tlearn: 0.0473009\ttotal: 45.8s\tremaining: 8.85s\n",
            "838:\tlearn: 0.0472754\ttotal: 45.9s\tremaining: 8.8s\n",
            "839:\tlearn: 0.0472570\ttotal: 45.9s\tremaining: 8.74s\n",
            "840:\tlearn: 0.0472321\ttotal: 46s\tremaining: 8.69s\n",
            "841:\tlearn: 0.0471917\ttotal: 46s\tremaining: 8.63s\n",
            "842:\tlearn: 0.0471456\ttotal: 46.1s\tremaining: 8.58s\n",
            "843:\tlearn: 0.0470825\ttotal: 46.1s\tremaining: 8.53s\n",
            "844:\tlearn: 0.0470574\ttotal: 46.2s\tremaining: 8.47s\n",
            "845:\tlearn: 0.0470198\ttotal: 46.2s\tremaining: 8.41s\n",
            "846:\tlearn: 0.0469822\ttotal: 46.3s\tremaining: 8.36s\n",
            "847:\tlearn: 0.0469530\ttotal: 46.3s\tremaining: 8.3s\n",
            "848:\tlearn: 0.0469096\ttotal: 46.4s\tremaining: 8.25s\n",
            "849:\tlearn: 0.0468539\ttotal: 46.5s\tremaining: 8.2s\n",
            "850:\tlearn: 0.0468223\ttotal: 46.5s\tremaining: 8.14s\n",
            "851:\tlearn: 0.0467712\ttotal: 46.6s\tremaining: 8.09s\n",
            "852:\tlearn: 0.0467374\ttotal: 46.6s\tremaining: 8.03s\n",
            "853:\tlearn: 0.0466958\ttotal: 46.7s\tremaining: 7.98s\n",
            "854:\tlearn: 0.0466582\ttotal: 46.7s\tremaining: 7.92s\n",
            "855:\tlearn: 0.0466317\ttotal: 46.8s\tremaining: 7.87s\n",
            "856:\tlearn: 0.0466275\ttotal: 46.8s\tremaining: 7.81s\n",
            "857:\tlearn: 0.0465961\ttotal: 46.9s\tremaining: 7.76s\n",
            "858:\tlearn: 0.0465454\ttotal: 46.9s\tremaining: 7.7s\n",
            "859:\tlearn: 0.0464962\ttotal: 47s\tremaining: 7.65s\n",
            "860:\tlearn: 0.0464438\ttotal: 47.1s\tremaining: 7.6s\n",
            "861:\tlearn: 0.0464075\ttotal: 47.1s\tremaining: 7.54s\n",
            "862:\tlearn: 0.0463576\ttotal: 47.2s\tremaining: 7.49s\n",
            "863:\tlearn: 0.0463086\ttotal: 47.2s\tremaining: 7.43s\n",
            "864:\tlearn: 0.0462713\ttotal: 47.3s\tremaining: 7.38s\n",
            "865:\tlearn: 0.0462142\ttotal: 47.3s\tremaining: 7.32s\n",
            "866:\tlearn: 0.0461779\ttotal: 47.4s\tremaining: 7.27s\n",
            "867:\tlearn: 0.0461634\ttotal: 47.4s\tremaining: 7.21s\n",
            "868:\tlearn: 0.0461496\ttotal: 47.5s\tremaining: 7.16s\n",
            "869:\tlearn: 0.0461222\ttotal: 47.5s\tremaining: 7.1s\n",
            "870:\tlearn: 0.0460936\ttotal: 47.6s\tremaining: 7.05s\n",
            "871:\tlearn: 0.0460590\ttotal: 47.6s\tremaining: 6.99s\n",
            "872:\tlearn: 0.0460290\ttotal: 47.7s\tremaining: 6.94s\n",
            "873:\tlearn: 0.0459910\ttotal: 47.8s\tremaining: 6.88s\n",
            "874:\tlearn: 0.0459573\ttotal: 47.8s\tremaining: 6.83s\n",
            "875:\tlearn: 0.0459191\ttotal: 47.9s\tremaining: 6.78s\n",
            "876:\tlearn: 0.0458937\ttotal: 47.9s\tremaining: 6.72s\n",
            "877:\tlearn: 0.0458502\ttotal: 48s\tremaining: 6.67s\n",
            "878:\tlearn: 0.0458193\ttotal: 48s\tremaining: 6.61s\n",
            "879:\tlearn: 0.0457728\ttotal: 48.1s\tremaining: 6.56s\n",
            "880:\tlearn: 0.0457357\ttotal: 48.1s\tremaining: 6.5s\n",
            "881:\tlearn: 0.0457161\ttotal: 48.2s\tremaining: 6.45s\n",
            "882:\tlearn: 0.0456835\ttotal: 48.3s\tremaining: 6.39s\n",
            "883:\tlearn: 0.0456586\ttotal: 48.3s\tremaining: 6.34s\n",
            "884:\tlearn: 0.0456516\ttotal: 48.4s\tremaining: 6.28s\n",
            "885:\tlearn: 0.0456168\ttotal: 48.4s\tremaining: 6.23s\n",
            "886:\tlearn: 0.0455690\ttotal: 48.5s\tremaining: 6.17s\n",
            "887:\tlearn: 0.0455492\ttotal: 48.5s\tremaining: 6.12s\n",
            "888:\tlearn: 0.0455251\ttotal: 48.6s\tremaining: 6.07s\n",
            "889:\tlearn: 0.0454983\ttotal: 48.6s\tremaining: 6.01s\n",
            "890:\tlearn: 0.0454717\ttotal: 48.7s\tremaining: 5.96s\n",
            "891:\tlearn: 0.0454402\ttotal: 48.7s\tremaining: 5.9s\n",
            "892:\tlearn: 0.0454142\ttotal: 48.8s\tremaining: 5.85s\n",
            "893:\tlearn: 0.0453758\ttotal: 48.9s\tremaining: 5.79s\n",
            "894:\tlearn: 0.0453006\ttotal: 48.9s\tremaining: 5.74s\n",
            "895:\tlearn: 0.0452636\ttotal: 49s\tremaining: 5.68s\n",
            "896:\tlearn: 0.0452199\ttotal: 49s\tremaining: 5.63s\n",
            "897:\tlearn: 0.0451949\ttotal: 49.1s\tremaining: 5.58s\n",
            "898:\tlearn: 0.0451636\ttotal: 49.1s\tremaining: 5.52s\n",
            "899:\tlearn: 0.0451567\ttotal: 49.2s\tremaining: 5.47s\n",
            "900:\tlearn: 0.0451014\ttotal: 49.2s\tremaining: 5.41s\n",
            "901:\tlearn: 0.0450648\ttotal: 49.3s\tremaining: 5.36s\n",
            "902:\tlearn: 0.0450461\ttotal: 49.4s\tremaining: 5.3s\n",
            "903:\tlearn: 0.0449924\ttotal: 49.4s\tremaining: 5.25s\n",
            "904:\tlearn: 0.0449346\ttotal: 49.5s\tremaining: 5.19s\n",
            "905:\tlearn: 0.0448762\ttotal: 49.5s\tremaining: 5.14s\n",
            "906:\tlearn: 0.0448652\ttotal: 49.6s\tremaining: 5.08s\n",
            "907:\tlearn: 0.0448244\ttotal: 49.6s\tremaining: 5.03s\n",
            "908:\tlearn: 0.0447955\ttotal: 49.7s\tremaining: 4.97s\n",
            "909:\tlearn: 0.0447744\ttotal: 49.7s\tremaining: 4.92s\n",
            "910:\tlearn: 0.0447310\ttotal: 49.8s\tremaining: 4.86s\n",
            "911:\tlearn: 0.0446875\ttotal: 49.9s\tremaining: 4.81s\n",
            "912:\tlearn: 0.0446658\ttotal: 49.9s\tremaining: 4.75s\n",
            "913:\tlearn: 0.0446441\ttotal: 50s\tremaining: 4.7s\n",
            "914:\tlearn: 0.0446129\ttotal: 50s\tremaining: 4.65s\n",
            "915:\tlearn: 0.0445810\ttotal: 50.1s\tremaining: 4.59s\n",
            "916:\tlearn: 0.0445414\ttotal: 50.1s\tremaining: 4.54s\n",
            "917:\tlearn: 0.0445073\ttotal: 50.2s\tremaining: 4.48s\n",
            "918:\tlearn: 0.0444638\ttotal: 50.2s\tremaining: 4.43s\n",
            "919:\tlearn: 0.0444525\ttotal: 50.3s\tremaining: 4.37s\n",
            "920:\tlearn: 0.0444238\ttotal: 50.3s\tremaining: 4.32s\n",
            "921:\tlearn: 0.0443784\ttotal: 50.4s\tremaining: 4.26s\n",
            "922:\tlearn: 0.0443465\ttotal: 50.4s\tremaining: 4.21s\n",
            "923:\tlearn: 0.0443263\ttotal: 50.5s\tremaining: 4.15s\n",
            "924:\tlearn: 0.0443154\ttotal: 50.5s\tremaining: 4.1s\n",
            "925:\tlearn: 0.0442642\ttotal: 50.6s\tremaining: 4.04s\n",
            "926:\tlearn: 0.0442460\ttotal: 50.6s\tremaining: 3.99s\n",
            "927:\tlearn: 0.0442330\ttotal: 50.7s\tremaining: 3.93s\n",
            "928:\tlearn: 0.0441967\ttotal: 50.8s\tremaining: 3.88s\n",
            "929:\tlearn: 0.0441735\ttotal: 50.8s\tremaining: 3.82s\n",
            "930:\tlearn: 0.0441489\ttotal: 50.9s\tremaining: 3.77s\n",
            "931:\tlearn: 0.0441294\ttotal: 50.9s\tremaining: 3.71s\n",
            "932:\tlearn: 0.0440815\ttotal: 51s\tremaining: 3.66s\n",
            "933:\tlearn: 0.0440445\ttotal: 51s\tremaining: 3.61s\n",
            "934:\tlearn: 0.0440230\ttotal: 51.1s\tremaining: 3.55s\n",
            "935:\tlearn: 0.0439711\ttotal: 51.1s\tremaining: 3.5s\n",
            "936:\tlearn: 0.0439453\ttotal: 51.2s\tremaining: 3.44s\n",
            "937:\tlearn: 0.0439005\ttotal: 51.3s\tremaining: 3.39s\n",
            "938:\tlearn: 0.0438778\ttotal: 51.3s\tremaining: 3.33s\n",
            "939:\tlearn: 0.0438339\ttotal: 51.4s\tremaining: 3.28s\n",
            "940:\tlearn: 0.0438109\ttotal: 51.4s\tremaining: 3.22s\n",
            "941:\tlearn: 0.0437883\ttotal: 51.5s\tremaining: 3.17s\n",
            "942:\tlearn: 0.0437441\ttotal: 51.5s\tremaining: 3.11s\n",
            "943:\tlearn: 0.0437160\ttotal: 51.6s\tremaining: 3.06s\n",
            "944:\tlearn: 0.0436813\ttotal: 51.6s\tremaining: 3s\n",
            "945:\tlearn: 0.0436572\ttotal: 51.7s\tremaining: 2.95s\n",
            "946:\tlearn: 0.0436375\ttotal: 51.7s\tremaining: 2.9s\n",
            "947:\tlearn: 0.0435891\ttotal: 51.8s\tremaining: 2.84s\n",
            "948:\tlearn: 0.0435712\ttotal: 51.8s\tremaining: 2.79s\n",
            "949:\tlearn: 0.0435299\ttotal: 51.9s\tremaining: 2.73s\n",
            "950:\tlearn: 0.0435089\ttotal: 52s\tremaining: 2.68s\n",
            "951:\tlearn: 0.0434808\ttotal: 52s\tremaining: 2.62s\n",
            "952:\tlearn: 0.0434526\ttotal: 52.1s\tremaining: 2.57s\n",
            "953:\tlearn: 0.0434360\ttotal: 52.1s\tremaining: 2.51s\n",
            "954:\tlearn: 0.0434164\ttotal: 52.2s\tremaining: 2.46s\n",
            "955:\tlearn: 0.0434086\ttotal: 52.2s\tremaining: 2.4s\n",
            "956:\tlearn: 0.0433831\ttotal: 52.3s\tremaining: 2.35s\n",
            "957:\tlearn: 0.0433746\ttotal: 52.3s\tremaining: 2.29s\n",
            "958:\tlearn: 0.0433291\ttotal: 52.4s\tremaining: 2.24s\n",
            "959:\tlearn: 0.0432961\ttotal: 52.4s\tremaining: 2.18s\n",
            "960:\tlearn: 0.0432548\ttotal: 52.5s\tremaining: 2.13s\n",
            "961:\tlearn: 0.0432225\ttotal: 52.5s\tremaining: 2.08s\n",
            "962:\tlearn: 0.0431747\ttotal: 52.6s\tremaining: 2.02s\n",
            "963:\tlearn: 0.0431610\ttotal: 52.7s\tremaining: 1.97s\n",
            "964:\tlearn: 0.0431077\ttotal: 52.7s\tremaining: 1.91s\n",
            "965:\tlearn: 0.0430831\ttotal: 52.8s\tremaining: 1.86s\n",
            "966:\tlearn: 0.0430618\ttotal: 52.8s\tremaining: 1.8s\n",
            "967:\tlearn: 0.0430145\ttotal: 52.9s\tremaining: 1.75s\n",
            "968:\tlearn: 0.0429927\ttotal: 52.9s\tremaining: 1.69s\n",
            "969:\tlearn: 0.0429536\ttotal: 53s\tremaining: 1.64s\n",
            "970:\tlearn: 0.0429351\ttotal: 53s\tremaining: 1.58s\n",
            "971:\tlearn: 0.0429067\ttotal: 53.1s\tremaining: 1.53s\n",
            "972:\tlearn: 0.0428793\ttotal: 53.2s\tremaining: 1.47s\n",
            "973:\tlearn: 0.0428626\ttotal: 53.2s\tremaining: 1.42s\n",
            "974:\tlearn: 0.0428567\ttotal: 53.3s\tremaining: 1.36s\n",
            "975:\tlearn: 0.0428314\ttotal: 53.3s\tremaining: 1.31s\n",
            "976:\tlearn: 0.0427863\ttotal: 53.4s\tremaining: 1.26s\n",
            "977:\tlearn: 0.0427458\ttotal: 53.4s\tremaining: 1.2s\n",
            "978:\tlearn: 0.0427184\ttotal: 53.5s\tremaining: 1.15s\n",
            "979:\tlearn: 0.0426809\ttotal: 53.5s\tremaining: 1.09s\n",
            "980:\tlearn: 0.0426357\ttotal: 53.6s\tremaining: 1.04s\n",
            "981:\tlearn: 0.0426090\ttotal: 53.6s\tremaining: 983ms\n",
            "982:\tlearn: 0.0425623\ttotal: 53.7s\tremaining: 929ms\n",
            "983:\tlearn: 0.0425449\ttotal: 53.8s\tremaining: 874ms\n",
            "984:\tlearn: 0.0425003\ttotal: 53.8s\tremaining: 819ms\n",
            "985:\tlearn: 0.0424760\ttotal: 53.9s\tremaining: 765ms\n",
            "986:\tlearn: 0.0424319\ttotal: 53.9s\tremaining: 710ms\n",
            "987:\tlearn: 0.0424250\ttotal: 54s\tremaining: 656ms\n",
            "988:\tlearn: 0.0424003\ttotal: 54s\tremaining: 601ms\n",
            "989:\tlearn: 0.0423859\ttotal: 54.1s\tremaining: 546ms\n",
            "990:\tlearn: 0.0423535\ttotal: 54.2s\tremaining: 492ms\n",
            "991:\tlearn: 0.0423196\ttotal: 54.2s\tremaining: 437ms\n",
            "992:\tlearn: 0.0422875\ttotal: 54.3s\tremaining: 383ms\n",
            "993:\tlearn: 0.0422565\ttotal: 54.3s\tremaining: 328ms\n",
            "994:\tlearn: 0.0422153\ttotal: 54.4s\tremaining: 273ms\n",
            "995:\tlearn: 0.0422051\ttotal: 54.4s\tremaining: 219ms\n",
            "996:\tlearn: 0.0421551\ttotal: 54.5s\tremaining: 164ms\n",
            "997:\tlearn: 0.0421190\ttotal: 54.5s\tremaining: 109ms\n",
            "998:\tlearn: 0.0420835\ttotal: 54.6s\tremaining: 54.7ms\n",
            "999:\tlearn: 0.0420526\ttotal: 54.7s\tremaining: 0us\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>catboost</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        catboost\n",
              "0              1\n",
              "1              0\n",
              "2              0\n",
              "3              0\n",
              "4              0\n",
              "...          ...\n",
              "142560         1\n",
              "142561         1\n",
              "142562         1\n",
              "142563         1\n",
              "142564         1\n",
              "\n",
              "[142565 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqHGqhF8iRgV"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "cb_test_pred.to_csv(\"cb_test_pred.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ovv6NCNhRPFr",
        "outputId": "6a97a41a-16a3-481f-dd62-518c4c142acd"
      },
      "source": [
        "cb_test_pred.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 298
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-7BaH2GTi2tV"
      },
      "source": [
        "##### ExtraTree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 464
        },
        "id": "6rRRMu4ri14j",
        "outputId": "087cfb23-ff47-4ec2-c0a3-74d0e2a428ab"
      },
      "source": [
        "ext_clf =  ExtraTreesClassifier(random_state=42)  #default n_estimators = 100, max_depth=None\r\n",
        "\r\n",
        "\r\n",
        "ext_params = {\r\n",
        "    'n_estimators' : [100, 1000]\r\n",
        "}\r\n",
        "\r\n",
        "ext_cv_tst = GridSearchCV(ext_clf, param_grid = ext_params, cv = 3)\r\n",
        "ext_cv_tst.fit(train_X, train_y)\r\n",
        "\r\n",
        "print('최적 하이퍼 파라미터:\\n', ext_cv_tst.best_params_)\r\n",
        "print('최고 예측 정확도: {0:.6f}'.format(ext_cv_tst.best_score_))\r\n",
        "\r\n",
        "joblib.dump(ext_cv_tst,'ext_cv_tst.pkl')\r\n",
        "\r\n",
        "\r\n",
        "# predict\r\n",
        "ext_pred_tst = ext_cv_tst.best_estimator_.predict(test_X)\r\n",
        "\r\n",
        "ext_test_pred = pd.DataFrame(ext_pred_tst, columns =['ExtraTree'])\r\n",
        "ext_test_pred\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "최적 하이퍼 파라미터:\n",
            " {'n_estimators': 1000}\n",
            "최고 예측 정확도: 0.896576\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ExtraTree</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        ExtraTree\n",
              "0               0\n",
              "1               0\n",
              "2               0\n",
              "3               0\n",
              "4               0\n",
              "...           ...\n",
              "142560          1\n",
              "142561          1\n",
              "142562          1\n",
              "142563          1\n",
              "142564          1\n",
              "\n",
              "[142565 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 150
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bl3ZE1YCjXDq"
      },
      "source": [
        "# submit\r\n",
        "\r\n",
        "ext_test_pred.to_csv(\"ext_test_pred.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_xlENgJjmZ_"
      },
      "source": [
        "##### LSTM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rn9tIUDfktUA",
        "outputId": "a53deeb9-23e9-43fa-8062-c7f2b8e82c85"
      },
      "source": [
        "# 실제 예측시 train\r\n",
        "index_seq = (train_X.argmax(axis=1)).argsort(axis=0)\r\n",
        "train_X = train_X[index_seq]\r\n",
        "train_y = train_y.iloc[index_seq]\r\n",
        "\r\n",
        "tf.random.set_seed(42)\r\n",
        "\r\n",
        "# 하이퍼파라미터\r\n",
        "max_epoch = 5\r\n",
        "batch_size = 32\r\n",
        "learning_rate = 0.001\r\n",
        "\r\n",
        "\r\n",
        "model = LSTM_add_feature(vocab_size, max_len = max_len, embedding_size = embedding_size)\r\n",
        "history = model.fit(x=[train_X[:,:-feature_num],train_X[:,-feature_num:]], y=train_y,epochs=max_epoch,batch_size = batch_size)\r\n",
        "\r\n",
        "model.save(\"lstm_stacking_test_model.h5\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_11 (InputLayer)           [(None, 20)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_5 (Embedding)         (None, 20, 300)      10139400    input_11[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout1d_2 (SpatialDro (None, 20, 300)      0           embedding_5[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "lstm_4 (LSTM)                   (None, 20, 32)       42624       spatial_dropout1d_2[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lstm_5 (LSTM)                   (None, 32)           8320        lstm_4[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "dropout_2 (Dropout)             (None, 32)           0           lstm_5[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "input_12 (InputLayer)           [(None, 4)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)     (None, 36)           0           dropout_2[0][0]                  \n",
            "                                                                 input_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 32)           1184        concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_5 (Dense)                 (None, 1)            33          dense_4[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 10,191,561\n",
            "Trainable params: 10,191,561\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Epoch 1/5\n",
            "3709/3709 [==============================] - 556s 148ms/step - loss: 0.0816 - accuracy: 0.9702\n",
            "Epoch 2/5\n",
            "3709/3709 [==============================] - 530s 143ms/step - loss: 0.0147 - accuracy: 0.9959\n",
            "Epoch 3/5\n",
            "3709/3709 [==============================] - 518s 140ms/step - loss: 0.0066 - accuracy: 0.9982\n",
            "Epoch 4/5\n",
            "3709/3709 [==============================] - 496s 134ms/step - loss: 0.0032 - accuracy: 0.9990\n",
            "Epoch 5/5\n",
            "3709/3709 [==============================] - 461s 124ms/step - loss: 0.0030 - accuracy: 0.9992\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "RrVFU0JKjmuP",
        "outputId": "33eb360e-fa18-4ae3-919a-c8241c080f61"
      },
      "source": [
        "# predict\r\n",
        "lstm_tst_model = keras.models.load_model(\"lstm_stacking_test_model.h5\")\r\n",
        "\r\n",
        "lstm_pred_tst = lstm_tst_model.predict([test_X[:,:max_len],test_X[:,-feature_num:]])\r\n",
        "lstm_pred_tst1 = lstm_pred_tst.reshape(-1)\r\n",
        "\r\n",
        "lstm_test_pred = pd.DataFrame(lstm_pred_tst1, columns =['LSTM'])\r\n",
        "lstm_test_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>LSTM</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5.922311e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.026399e-05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3.369693e-06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3.632282e-07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4.333108e-07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>9.999997e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                LSTM\n",
              "0       5.922311e-01\n",
              "1       1.026399e-05\n",
              "2       3.369693e-06\n",
              "3       3.632282e-07\n",
              "4       4.333108e-07\n",
              "...              ...\n",
              "142560  9.999997e-01\n",
              "142561  1.000000e+00\n",
              "142562  1.000000e+00\n",
              "142563  1.000000e+00\n",
              "142564  1.000000e+00\n",
              "\n",
              "[142565 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 161
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5pI9t_0DkQJ8"
      },
      "source": [
        "# submit\r\n",
        "lstm_test_pred.to_csv(\"lstm_test_pred.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xHzJqErgFEMm"
      },
      "source": [
        "##### 스태킹"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nAstYDF2Op_h"
      },
      "source": [
        "- train_X로 학습한 모델들 로드."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2V4xQS9JYUBC",
        "outputId": "f419695f-6f3b-46db-97a5-b13c7acb6fe0"
      },
      "source": [
        "gb_clf_tr = GradientBoostingClassifier(learning_rate = 0.1, n_estimators = 1000, random_state = 42) # 기본값: max_depth=3, learning_rate=0.1\r\n",
        "gb_clf_tr.fit(train_X, train_y)\r\n",
        "\r\n",
        "joblib.dump(gb_clf_tr,'gb_clf_tr.pkl')"
      ],
      "execution_count": 326,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['gb_clf_tr.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 326
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ho-CFby9IAlJ",
        "outputId": "1bb2df8b-fa74-4a60-8f0d-70b420d030c7"
      },
      "source": [
        "ext_clf =  ExtraTreesClassifier(random_state=42) \r\n",
        "ext_clf.fit(train_X, train_y)"
      ],
      "execution_count": 264,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
              "                     criterion='gini', max_depth=None, max_features='auto',\n",
              "                     max_leaf_nodes=None, max_samples=None,\n",
              "                     min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                     min_samples_leaf=1, min_samples_split=2,\n",
              "                     min_weight_fraction_leaf=0.0, n_estimators=100,\n",
              "                     n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
              "                     warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 264
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CQP3MoOVYs9D",
        "outputId": "e5999fa1-769c-48ac-e01e-e4e7b805cb47"
      },
      "source": [
        "joblib.dump(ext_clf,'ext_clf_tr.pkl')"
      ],
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ext_clf_tr.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        },
        "id": "Kk4XkFOz96Wn",
        "outputId": "6213ff7c-39b8-4ac0-916e-6fac8a4208c3"
      },
      "source": [
        "xgb_cl=XGBClassifier(random_state=42)\r\n",
        "xgb_params={'objective':['binary:logistic'],'booster':['gbtree'],'n_estimators':[100,500,1000],'learning_rate':[0.001,0.05,0.1],'max_depth':[2,4,8]}\r\n",
        "grid_xgb=GridSearchCV(xgb_cl,param_grid=xgb_params,cv=3)\r\n",
        "grid_xgb.fit(train_X,train_y)"
      ],
      "execution_count": 376,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-376-d7039619fa02>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mxgb_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'objective'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'binary:logistic'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'booster'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'gbtree'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'n_estimators'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'learning_rate'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0.001\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.05\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'max_depth'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mgrid_xgb\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mGridSearchCV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxgb_cl\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mxgb_params\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mgrid_xgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrain_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    708\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 710\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1149\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1150\u001b[0m         \u001b[0;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1151\u001b[0;31m         \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    687\u001b[0m                                \u001b[0;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m                                in product(candidate_params,\n\u001b[0;32m--> 689\u001b[0;31m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1042\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1044\u001b[0;31m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1045\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1046\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    857\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    858\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 859\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    860\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    861\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    775\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    776\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 777\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    778\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    206\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    209\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    570\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 572\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    574\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 263\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 263\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    513\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    514\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 515\u001b[0;31m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    516\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    517\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/xgboost/sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, callbacks)\u001b[0m\n\u001b[1;32m    730\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    731\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxgb_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mxgb_model\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 732\u001b[0;31m                               callbacks=callbacks)\n\u001b[0m\u001b[1;32m    733\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    734\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobjective\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb_options\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"objective\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/xgboost/training.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[1;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/xgboost/training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/xgboost/core.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[1;32m   1107\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[0;32m-> 1109\u001b[0;31m                                                     dtrain.handle))\n\u001b[0m\u001b[1;32m   1110\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m             \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        },
        "id": "41eTc7jkHEHh",
        "outputId": "bc76fcfe-e514-4604-974b-e2acce341f0d"
      },
      "source": [
        "from sklearn.externals import joblib \r\n",
        "\r\n",
        "path = \"/content/drive/MyDrive/Colab Notebooks/dacon_news/테스트 stacking1\"\r\n",
        "\r\n",
        "\r\n",
        "# gb_grid_tst = joblib.load(path + \"/gb_grid_tst.pkl\")\r\n",
        "\r\n",
        "# cb_clf_tst = joblib.load(path +  \"/cb_clf_tst.pkl\")\r\n",
        "\r\n",
        "# ada_grid_tst = joblib.load(path +  \"/ada_grid_tst.pkl\")\r\n"
      ],
      "execution_count": 325,
      "outputs": [
        {
          "output_type": "error",
          "ename": "EOFError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mEOFError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-325-dfdd2d058105>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mgb_grid_tst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/gb_grid_tst.pkl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m# cb_clf_tst = joblib.load(path +  \"/cb_clf_tst.pkl\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(filename, mmap_mode)\u001b[0m\n\u001b[1;32m    583\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mload_compatibility\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    584\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 585\u001b[0;31m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_unpickle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmmap_mode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    586\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36m_unpickle\u001b[0;34m(fobj, filename, mmap_mode)\u001b[0m\n\u001b[1;32m    502\u001b[0m     \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m         \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0munpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m             warnings.warn(\"The file '%s' has been generated with a \"\n",
            "\u001b[0;32m/usr/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1046\u001b[0m                 \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1047\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1048\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mEOFError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1049\u001b[0m                 \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbytes_types\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1050\u001b[0m                 \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mEOFError\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2JinE4GI8NY"
      },
      "source": [
        "path = \"/content/drive/MyDrive/Colab Notebooks/dacon_news/테스트 stacking1\"\r\n",
        "lstm_tst_model = keras.models.load_model(path + \"/lstm_stacking_test_model.h5\")\r\n"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o2BSf2KHZGvm"
      },
      "source": [
        "-  train dataset 예측"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRo-kRv9-of7"
      },
      "source": [
        "\r\n",
        "pred_xgb=grid_xgb.predict(train_X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OCyVZbOBZAOb"
      },
      "source": [
        "gb_pred_tr = gb_clf_tr.predict(train_X)"
      ],
      "execution_count": 328,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PKtrX9S4NjtW"
      },
      "source": [
        "ada_pred_tr = ada_grid_tst.predict(train_X)\r\n",
        "cb_pred_tr = cb_clf_tst.predict(train_X)\r\n",
        "ext_pred_tr = ext_clf.predict(train_X)"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LIDRICVjw0kH"
      },
      "source": [
        "ext_pred_tr1 = ext_clf.predict(train_X)"
      ],
      "execution_count": 265,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pClBBY7IUn-n"
      },
      "source": [
        "lstm_pred_tr = lstm_tst_model.predict([train_X[:,:max_len],train_X[:,-feature_num:]])\r\n",
        "lstm_pred_tr1 = lstm_pred_tr.reshape(-1)"
      ],
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-0FhiYaUhmYr",
        "outputId": "ca9ae758-1010-46e0-cc6c-658294a48bed"
      },
      "source": [
        "lstm_pred_tr1.shape"
      ],
      "execution_count": 161,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(118676,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 161
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lTZTzNUHVPc-",
        "outputId": "8b1cf4a5-5ae0-4639-bd5e-6ec6d800a45f"
      },
      "source": [
        "rnn_pred_tr = np.array(pd.read_csv('rnn_predicted (1).csv')).reshape(-1)\r\n",
        "rnn_pred_tr.shape"
      ],
      "execution_count": 205,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(118745,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 205
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "ybOs4zK1m30-",
        "outputId": "da7bfd1d-b811-4d60-b101-117d98a83ff3"
      },
      "source": [
        "a = pd.read_csv('rnn_predicted (1).csv')\r\n",
        "a"
      ],
      "execution_count": 206,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.000021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.999981</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.999972</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.000002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.000055</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118740</th>\n",
              "      <td>0.999999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118741</th>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118742</th>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118743</th>\n",
              "      <td>0.999994</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118744</th>\n",
              "      <td>0.999995</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>118745 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "               0\n",
              "0       0.000021\n",
              "1       0.999981\n",
              "2       0.999972\n",
              "3       0.000002\n",
              "4       0.000055\n",
              "...          ...\n",
              "118740  0.999999\n",
              "118741  1.000000\n",
              "118742  1.000000\n",
              "118743  0.999994\n",
              "118744  0.999995\n",
              "\n",
              "[118745 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 206
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JbULeSKpFEoq",
        "outputId": "ea134f3a-c1e7-4a16-9463-facda36555e4"
      },
      "source": [
        "train_all_pred = np.array([ada_pred_tr, cb_pred_tr, ext_pred_tr1, lstm_pred_tr1, gb_pred_tr, pred_xgb])\r\n",
        "train_all_pred = np.transpose(train_all_pred)\r\n",
        "train_all_pred.shape\r\n"
      ],
      "execution_count": 329,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(118676, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 329
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w1wIes5AXqWw"
      },
      "source": [
        "- test 예측값 로드"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EmNH3n2j6ZMB"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6BumwpCvw-wl",
        "outputId": "49481158-1b59-46d5-c54c-3ad93b21130f"
      },
      "source": [
        "ext_pred_tst1 = ext_clf.predict(test_X)\r\n",
        "ext_pred_tst1.shape"
      ],
      "execution_count": 281,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 281
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VeEyrc3JXcFg"
      },
      "source": [
        "gb_pred_tst = np.array(pd.read_csv(path+\"/gb_test_predict.csv\")).reshape(-1)\r\n",
        "ada_pred_tst = np.array(pd.read_csv(path+\"/ada_test_pred.csv\")).reshape(-1)\r\n",
        "cb_pred_tst = np.array(pd.read_csv(path+\"/cb_test_pred.csv\")).reshape(-1)\r\n",
        "ext_pred_tst = np.array(pd.read_csv(path+\"/ext_test_pred.csv\")).reshape(-1)\r\n",
        "lstm_pred_tst = np.array(pd.read_csv(path+\"/lstm_test_pred.csv\")).reshape(-1)"
      ],
      "execution_count": 332,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zInCmxuBfbVF",
        "outputId": "b850ee76-edbe-488c-cef0-e4d7c96c1da1"
      },
      "source": [
        "rnn_tst = pd.read_csv('submission (2).csv')\r\n",
        "rnn_tst = rnn_tst['info']\r\n",
        "rnn_tst.shape"
      ],
      "execution_count": 327,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 327
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-ODkeT_8ZU81",
        "outputId": "751c0060-502c-4078-a975-6a82f7574c11"
      },
      "source": [
        "test_all_pred = np.array([ada_pred_tst, cb_pred_tst, ext_pred_tst1, lstm_pred_tst, gb_pred_tst])\r\n",
        "test_all_pred = np.transpose(test_all_pred)\r\n",
        "test_all_pred.shape\r\n"
      ],
      "execution_count": 333,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 333
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YZUm1pMq7L6v",
        "outputId": "2aa7669b-31c8-47e6-ca54-ff7cca98e226"
      },
      "source": [
        "train_all_pred.shape"
      ],
      "execution_count": 342,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(118676, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 342
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2uSi_oguFjpB",
        "outputId": "021b2a58-b662-4b61-8429-5fad9e746be7"
      },
      "source": [
        "test_lgbm_clf=LGBMClassifier(random_state=42)\r\n",
        "test_lgbm_clf.fit(train_all_pred, train_y)\r\n",
        "\r\n",
        "# joblib.dump(test_lgbm_clf,'train_lgbm_stacking_clf.pkl')\r\n",
        "\r\n",
        "test_lgbm_pred = test_lgbm_clf.predict(test_all_pred, max_depth=2) #all_pred\r\n",
        "test_lgbm_pred\r\n"
      ],
      "execution_count": 368,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, ..., 1, 1, 1], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 368
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vL-XQbuH0bG4",
        "outputId": "227a8a41-ea1d-4ca6-824e-ae816976f619"
      },
      "source": [
        "test_lgbm_clf.get_params"
      ],
      "execution_count": 369,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<bound method LGBMModel.get_params of LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
              "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
              "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
              "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
              "               random_state=42, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
              "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 369
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p9pNi4pscFry"
      },
      "source": [
        "path = \"/content/drive/MyDrive/Colab Notebooks/dacon_news\"\r\n",
        "\r\n",
        "submission = pd.read_csv(path+\"/data/sample_submission.csv\")\r\n",
        "submission[\"info\"]=test_lgbm_pred\r\n",
        "submission.to_csv(\"stacking_submission2.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": 370,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "4x7XRcPjdNJg",
        "outputId": "89a9f3a2-f894-42f0-8580-79effac798dd"
      },
      "source": [
        "submission"
      ],
      "execution_count": 371,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>info</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NEWS00237_1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NEWS00237_2</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NEWS00237_3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NEWS00237_4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NEWS00237_5</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>NEWS09482_72</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>NEWS09482_73</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>NEWS09482_74</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>NEWS09482_75</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>NEWS09482_76</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                  id  info\n",
              "0        NEWS00237_1     0\n",
              "1        NEWS00237_2     0\n",
              "2        NEWS00237_3     0\n",
              "3        NEWS00237_4     0\n",
              "4        NEWS00237_5     0\n",
              "...              ...   ...\n",
              "142560  NEWS09482_72     1\n",
              "142561  NEWS09482_73     1\n",
              "142562  NEWS09482_74     1\n",
              "142563  NEWS09482_75     1\n",
              "142564  NEWS09482_76     1\n",
              "\n",
              "[142565 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 371
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "F8QOG7qJdwx5",
        "outputId": "13756f5a-3166-4605-c091-011240ac1f76"
      },
      "source": [
        "# sub_fin = pd.read_csv('submission (2).csv')\r\n",
        "sub_fin"
      ],
      "execution_count": 372,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>info</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NEWS00237_1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NEWS00237_2</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NEWS00237_3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NEWS00237_4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NEWS00237_5</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>NEWS09482_72</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>NEWS09482_73</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>NEWS09482_74</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>NEWS09482_75</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>NEWS09482_76</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                  id  info\n",
              "0        NEWS00237_1     0\n",
              "1        NEWS00237_2     0\n",
              "2        NEWS00237_3     0\n",
              "3        NEWS00237_4     0\n",
              "4        NEWS00237_5     0\n",
              "...              ...   ...\n",
              "142560  NEWS09482_72     1\n",
              "142561  NEWS09482_73     1\n",
              "142562  NEWS09482_74     1\n",
              "142563  NEWS09482_75     1\n",
              "142564  NEWS09482_76     1\n",
              "\n",
              "[142565 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 372
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XTkD1lWidWPy",
        "outputId": "4bc504d6-5342-4b82-fcd6-17a30ecbb4a4"
      },
      "source": [
        "len(submission[sub_fin['info']!=submission['info']])"
      ],
      "execution_count": 373,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4966"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 373
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zqYdPdwEkx7p",
        "outputId": "b6efcc22-d783-4346-b926-ecf8491d58a9"
      },
      "source": [
        "len(submission[submission['info']==1])"
      ],
      "execution_count": 359,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "51787"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 359
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U3Ghet-Dwiau"
      },
      "source": [
        "##### stratified CV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FjGiO4Djwkm9",
        "outputId": "6470cf20-dab3-4e34-8f46-97e6a0bb7c62"
      },
      "source": [
        "\r\n",
        "# train\r\n",
        "stk_trn = [ada_pred_tr, cb_pred_tr, ext_pred_tr, lstm_pred_tr1]\r\n",
        "# test\r\n",
        "stk_tst = [ada_pred_tst, cb_pred_tst, ext_pred_tst, lstm_pred_tst]\r\n",
        "feature_names = []\r\n",
        "model_names = ['ada','cb','ext','lstm']\r\n",
        "\r\n",
        "for model in model_names:\r\n",
        "    feature_names += [f'{model}_class0', f'{model}_class1']\r\n",
        "    \r\n",
        "stk_trn = np.hstack(stk_trn)\r\n",
        "stk_tst = np.hstack(stk_tst)\r\n",
        "\r\n",
        "feature_names"
      ],
      "execution_count": 207,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ada_class0',\n",
              " 'ada_class1',\n",
              " 'cb_class0',\n",
              " 'cb_class1',\n",
              " 'ext_class0',\n",
              " 'ext_class1',\n",
              " 'lstm_class0',\n",
              " 'lstm_class1']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 207
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TRL9QHU8y9SV",
        "outputId": "7e493733-08ff-4a8b-f418-3814648ee8b4"
      },
      "source": [
        "stk_trn = stk_trn.reshape(-1,1)\r\n",
        "stk_tst = stk_tst.reshape(-1, 1)\r\n",
        "\r\n",
        "stk_trn.shape"
      ],
      "execution_count": 220,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(474704, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 220
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NVskVLifzYL5",
        "outputId": "9d294e88-2862-4529-fc17-37925f503d90"
      },
      "source": [
        "y =[train_y, train_y, train_y, train_y]\r\n",
        "y = np.hstack(y).reshape(-1, 1)\r\n",
        "y.shape"
      ],
      "execution_count": 221,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(474704, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 221
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XhxsP7Dip1S5",
        "outputId": "f6d1623f-dc43-4c54-88da-3512c1a471ce"
      },
      "source": [
        "a = y.reshape(-1)\r\n",
        "a.shape"
      ],
      "execution_count": 228,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(474704,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 228
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HT9M03TJp9Jk",
        "outputId": "267aa162-e054-4f9e-eb78-e5d31e972ae3"
      },
      "source": [
        "a"
      ],
      "execution_count": 230,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 1, ..., 1, 1, 1], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 230
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7qtwKECEp_L6",
        "outputId": "83615c42-1dc8-4213-9410-c5b34479a442"
      },
      "source": [
        "stk_trn"
      ],
      "execution_count": 232,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.],\n",
              "       [1.],\n",
              "       [1.],\n",
              "       ...,\n",
              "       [1.],\n",
              "       [1.],\n",
              "       [1.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 232
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JLpjd1kfo_ao",
        "outputId": "3e7a8a0a-b412-4e97-e572-b7236a3d8f44"
      },
      "source": [
        "p_val = np.zeros((stk_trn.shape[0], 1))\r\n",
        "p_tst = np.zeros((stk_tst.shape[0], 1))\r\n",
        "p_val.shape"
      ],
      "execution_count": 217,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(474704, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 217
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jq3U84Djp7-u",
        "outputId": "99a1b175-a373-4bc4-d539-bb262ece4db0"
      },
      "source": [
        "p_val"
      ],
      "execution_count": 250,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.],\n",
              "       [0.],\n",
              "       [0.],\n",
              "       ...,\n",
              "       [0.],\n",
              "       [0.],\n",
              "       [0.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 250
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7suWIRe8sJDE",
        "outputId": "1b697e81-84fd-4e8d-cb06-72c402995787"
      },
      "source": [
        "test_lgbm_pred.reshape(-1,1)"
      ],
      "execution_count": 254,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0],\n",
              "       [0],\n",
              "       [0],\n",
              "       ...,\n",
              "       [1],\n",
              "       [1],\n",
              "       [1]], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 254
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "UQzI7jB8wkqa",
        "outputId": "c90ca488-63d6-4891-b194-a1e543e97566"
      },
      "source": [
        "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\r\n",
        "\r\n",
        "p_val = np.zeros((stk_trn.shape[0], 1))\r\n",
        "p_tst = np.zeros((stk_tst.shape[0], 1))\r\n",
        "for i, (i_trn, i_val) in enumerate(cv.split(stk_trn, y), 1):\r\n",
        "    print(f'training model for CV #{i}')\r\n",
        "    test_lgbm_clf = LGBMClassifier(random_state=42)\r\n",
        "    test_lgbm_clf.fit(stk_trn[i_trn], y[i_trn],eval_set=[(stk_trn[i_val], y[i_val])], verbose=True)\r\n",
        "           \r\n",
        "    p_val[i_val, :] = (test_lgbm_clf.predict_proba(stk_trn[i_val].reshape)).reshape(-1,1)\r\n",
        "    p_tst += test_lgbm_clf.predict_proba(stk_tst) / n_fold\r\n",
        "    print(accuracy_score(y[i_val],np.argmax(p_val[i_val],axis=1)))\r\n",
        "print()\r\n",
        "print('models:',model_names)\r\n",
        "print(test_lgbm_clf)\r\n",
        "\r\n",
        "print(f'valid cv accuracy : {accuracy_score(y, np.argmax(p_val, axis=1)) :.6f}')\r\n",
        "print(f' {confusion_matrix(y, np.argmax(p_val, axis=1))}')"
      ],
      "execution_count": 249,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "training model for CV #1\n",
            "[1]\tvalid_0's binary_logloss: 0.57889\n",
            "[2]\tvalid_0's binary_logloss: 0.504672\n",
            "[3]\tvalid_0's binary_logloss: 0.443451\n",
            "[4]\tvalid_0's binary_logloss: 0.392087\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_label.py:235: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_label.py:268: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[5]\tvalid_0's binary_logloss: 0.348456\n",
            "[6]\tvalid_0's binary_logloss: 0.311045\n",
            "[7]\tvalid_0's binary_logloss: 0.278731\n",
            "[8]\tvalid_0's binary_logloss: 0.250656\n",
            "[9]\tvalid_0's binary_logloss: 0.226149\n",
            "[10]\tvalid_0's binary_logloss: 0.204672\n",
            "[11]\tvalid_0's binary_logloss: 0.185793\n",
            "[12]\tvalid_0's binary_logloss: 0.169153\n",
            "[13]\tvalid_0's binary_logloss: 0.154454\n",
            "[14]\tvalid_0's binary_logloss: 0.141448\n",
            "[15]\tvalid_0's binary_logloss: 0.129923\n",
            "[16]\tvalid_0's binary_logloss: 0.119699\n",
            "[17]\tvalid_0's binary_logloss: 0.11062\n",
            "[18]\tvalid_0's binary_logloss: 0.102554\n",
            "[19]\tvalid_0's binary_logloss: 0.0953828\n",
            "[20]\tvalid_0's binary_logloss: 0.0890066\n",
            "[21]\tvalid_0's binary_logloss: 0.0833364\n",
            "[22]\tvalid_0's binary_logloss: 0.0782941\n",
            "[23]\tvalid_0's binary_logloss: 0.0738112\n",
            "[24]\tvalid_0's binary_logloss: 0.0698268\n",
            "[25]\tvalid_0's binary_logloss: 0.0662871\n",
            "[26]\tvalid_0's binary_logloss: 0.0631444\n",
            "[27]\tvalid_0's binary_logloss: 0.060356\n",
            "[28]\tvalid_0's binary_logloss: 0.0578839\n",
            "[29]\tvalid_0's binary_logloss: 0.0556942\n",
            "[30]\tvalid_0's binary_logloss: 0.0537565\n",
            "[31]\tvalid_0's binary_logloss: 0.0520436\n",
            "[32]\tvalid_0's binary_logloss: 0.0505311\n",
            "[33]\tvalid_0's binary_logloss: 0.0491972\n",
            "[34]\tvalid_0's binary_logloss: 0.0480221\n",
            "[35]\tvalid_0's binary_logloss: 0.0469882\n",
            "[36]\tvalid_0's binary_logloss: 0.0460798\n",
            "[37]\tvalid_0's binary_logloss: 0.0452826\n",
            "[38]\tvalid_0's binary_logloss: 0.0445839\n",
            "[39]\tvalid_0's binary_logloss: 0.0439723\n",
            "[40]\tvalid_0's binary_logloss: 0.0434374\n",
            "[41]\tvalid_0's binary_logloss: 0.0429703\n",
            "[42]\tvalid_0's binary_logloss: 0.0425627\n",
            "[43]\tvalid_0's binary_logloss: 0.0422075\n",
            "[44]\tvalid_0's binary_logloss: 0.041898\n",
            "[45]\tvalid_0's binary_logloss: 0.0416287\n",
            "[46]\tvalid_0's binary_logloss: 0.0413944\n",
            "[47]\tvalid_0's binary_logloss: 0.0411907\n",
            "[48]\tvalid_0's binary_logloss: 0.0410136\n",
            "[49]\tvalid_0's binary_logloss: 0.0408597\n",
            "[50]\tvalid_0's binary_logloss: 0.0407259\n",
            "[51]\tvalid_0's binary_logloss: 0.0406095\n",
            "[52]\tvalid_0's binary_logloss: 0.0405083\n",
            "[53]\tvalid_0's binary_logloss: 0.0404202\n",
            "[54]\tvalid_0's binary_logloss: 0.0403435\n",
            "[55]\tvalid_0's binary_logloss: 0.0402767\n",
            "[56]\tvalid_0's binary_logloss: 0.0402184\n",
            "[57]\tvalid_0's binary_logloss: 0.0401674\n",
            "[58]\tvalid_0's binary_logloss: 0.040123\n",
            "[59]\tvalid_0's binary_logloss: 0.040084\n",
            "[60]\tvalid_0's binary_logloss: 0.0400499\n",
            "[61]\tvalid_0's binary_logloss: 0.04002\n",
            "[62]\tvalid_0's binary_logloss: 0.0399938\n",
            "[63]\tvalid_0's binary_logloss: 0.0399707\n",
            "[64]\tvalid_0's binary_logloss: 0.0399505\n",
            "[65]\tvalid_0's binary_logloss: 0.0399326\n",
            "[66]\tvalid_0's binary_logloss: 0.0399169\n",
            "[67]\tvalid_0's binary_logloss: 0.0399031\n",
            "[68]\tvalid_0's binary_logloss: 0.0398909\n",
            "[69]\tvalid_0's binary_logloss: 0.0398801\n",
            "[70]\tvalid_0's binary_logloss: 0.0398706\n",
            "[71]\tvalid_0's binary_logloss: 0.0398623\n",
            "[72]\tvalid_0's binary_logloss: 0.0398549\n",
            "[73]\tvalid_0's binary_logloss: 0.0398484\n",
            "[74]\tvalid_0's binary_logloss: 0.0398427\n",
            "[75]\tvalid_0's binary_logloss: 0.0398377\n",
            "[76]\tvalid_0's binary_logloss: 0.0398333\n",
            "[77]\tvalid_0's binary_logloss: 0.0398295\n",
            "[78]\tvalid_0's binary_logloss: 0.0398262\n",
            "[79]\tvalid_0's binary_logloss: 0.0398233\n",
            "[80]\tvalid_0's binary_logloss: 0.0398208\n",
            "[81]\tvalid_0's binary_logloss: 0.0398187\n",
            "[82]\tvalid_0's binary_logloss: 0.0398168\n",
            "[83]\tvalid_0's binary_logloss: 0.0398153\n",
            "[84]\tvalid_0's binary_logloss: 0.039814\n",
            "[85]\tvalid_0's binary_logloss: 0.0398129\n",
            "[86]\tvalid_0's binary_logloss: 0.0398121\n",
            "[87]\tvalid_0's binary_logloss: 0.0398114\n",
            "[88]\tvalid_0's binary_logloss: 0.0398109\n",
            "[89]\tvalid_0's binary_logloss: 0.0398106\n",
            "[90]\tvalid_0's binary_logloss: 0.0398104\n",
            "[91]\tvalid_0's binary_logloss: 0.0398103\n",
            "[92]\tvalid_0's binary_logloss: 0.0398103\n",
            "[93]\tvalid_0's binary_logloss: 0.0398104\n",
            "[94]\tvalid_0's binary_logloss: 0.0398106\n",
            "[95]\tvalid_0's binary_logloss: 0.0398109\n",
            "[96]\tvalid_0's binary_logloss: 0.0398113\n",
            "[97]\tvalid_0's binary_logloss: 0.0398117\n",
            "[98]\tvalid_0's binary_logloss: 0.0398122\n",
            "[99]\tvalid_0's binary_logloss: 0.0398127\n",
            "[100]\tvalid_0's binary_logloss: 0.0398133\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-249-8ecd9c318dc8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mtest_lgbm_clf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstk_trn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_trn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_trn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0meval_set\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstk_trn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_val\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_val\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mp_val\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtest_lgbm_clf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstk_trn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_val\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mp_tst\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtest_lgbm_clf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstk_tst\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mn_fold\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_val\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp_val\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_val\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/lightgbm/sklearn.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[0;34m(self, X, raw_score, num_iteration, pred_leaf, pred_contrib, **kwargs)\u001b[0m\n\u001b[1;32m    798\u001b[0m         \"\"\"\n\u001b[1;32m    799\u001b[0m         result = super(LGBMClassifier, self).predict(X, raw_score, num_iteration,\n\u001b[0;32m--> 800\u001b[0;31m                                                      pred_leaf, pred_contrib, **kwargs)\n\u001b[0m\u001b[1;32m    801\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_classes\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mraw_score\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mpred_leaf\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mpred_contrib\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    802\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/lightgbm/sklearn.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X, raw_score, num_iteration, pred_leaf, pred_contrib, **kwargs)\u001b[0m\n\u001b[1;32m    597\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLGBMNotFittedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Estimator not fitted, call `fit` before exploiting the model.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataTable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_LGBMCheckArray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforce_all_finite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m         \u001b[0mn_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_features\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mn_features\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    547\u001b[0m                     \u001b[0;34m\"Reshape your data either using array.reshape(-1, 1) if \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m                     \u001b[0;34m\"your data has a single feature or array.reshape(1, -1) \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 549\u001b[0;31m                     \"if it contains a single sample.\".format(array))\n\u001b[0m\u001b[1;32m    550\u001b[0m             \u001b[0;31m# If input is 1D raise error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Expected 2D array, got scalar array instead:\narray=<built-in method reshape of numpy.ndarray object at 0x7fe84716e2b0>.\nReshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample."
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VxxA4BtOWqwR"
      },
      "source": [
        "##### 모델 로드"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xf1AWyn4WtY-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "da18fe13-35b4-4e7e-a946-a9464bba2e35"
      },
      "source": [
        "#path = \"/content/drive/MyDrive/Colab Notebooks/dacon_news/테스트 stacking1\"\r\n",
        "\r\n",
        "#with open(\"/ada_grid_tst.pkl\", 'wb') as f:\r\n",
        "#  pickle.dump(ada_grid_tst, f)\r\n"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-1f4239b9330d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/ada_grid_tst.pkl\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m   \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mada_grid_tst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'ada_grid_tst' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZd1uakF1YXv"
      },
      "source": [
        "### cv"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ewUcnFfg4ECh"
      },
      "source": [
        "\r\n",
        "gb_pred_tst = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/fake news detection/테스트 stacking1/gb_test_predict.csv')\r\n",
        "ada_pred_tst= pd.read_csv('/content/drive/MyDrive/Colab Notebooks/fake news detection/테스트 stacking1/ada_test_pred.csv')\r\n",
        "cb_pred_tst= pd.read_csv('/content/drive/MyDrive/Colab Notebooks/fake news detection/테스트 stacking1/cb_test_pred.csv')\r\n",
        "ext_pred_tst = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/fake news detection/테스트 stacking1/ext_test_pred.csv')\r\n",
        "lstm_pred_tst1 = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/fake news detection/테스트 stacking1/lstm_test_pred.csv')\r\n",
        "\r\n",
        "gb_pred_tst = np.array(gb_pred_tst).reshape(-1)\r\n",
        "ada_pred_tst = np.array(ada_pred_tst).reshape(-1)\r\n",
        "cb_pred_tst = np.array(cb_pred_tst).reshape(-1)\r\n",
        "ext_pred_tst = np.array(ext_pred_tst).reshape(-1)\r\n",
        "lstm_pred_tst1 = np.array(lstm_pred_tst1).reshape(-1)\r\n",
        "\r\n",
        "test_all_pred = np.array([gb_pred_tst, ada_pred_tst, cb_pred_tst, ext_pred_tst, lstm_pred_tst1])\r\n",
        "test_all_pred = np.transpose(test_all_pred)\r\n",
        "test_all_pred.shape\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GIfXXPnGsgcW"
      },
      "source": [
        "from sklearn.model_selection import KFold\r\n",
        "from sklearn.metrics import mean_absolute_error\r\n",
        "\r\n",
        "# 개별 기반 모델에서 최종 메타 모델이 사용할 학습 및 테스트용 데이터를 생성하기 위한 함수. \r\n",
        "def get_stacking_base_datasets(model, X_train_n, y_train_n, X_test_n, n_folds ):\r\n",
        "\r\n",
        "    kf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\r\n",
        "    #추후에 메타 모델이 사용할 학습 데이터 반환을 위한 넘파이 배열 초기화 \r\n",
        "    train_fold_pred = np.zeros((X_train_n.shape[0] ,1 ))\r\n",
        "    test_pred = np.zeros((X_test_n.shape[0],n_folds))\r\n",
        "    print(model.__class__.__name__ , ' model 시작 ')\r\n",
        "    \r\n",
        "    for folder_counter , (train_index, valid_index) in enumerate(kf.split(X_train_n)):\r\n",
        "        #입력된 학습 데이터에서 기반 모델이 학습/예측할 폴드 데이터 셋 추출 \r\n",
        "        print('\\t 폴드 세트: ',folder_counter,' 시작 ')\r\n",
        "        X_tr = X_train_n[train_index] \r\n",
        "        y_tr = y_train_n[train_index] \r\n",
        "        X_te = X_train_n[valid_index]  \r\n",
        "        \r\n",
        "        #폴드 세트 내부에서 다시 만들어진 학습 데이터로 기반 모델의 학습 수행.\r\n",
        "        model.fit(X_tr , y_tr)       \r\n",
        "        #폴드 세트 내부에서 다시 만들어진 검증 데이터로 기반 모델 예측 후 데이터 저장.\r\n",
        "        train_fold_pred[valid_index, :] = model.predict(X_te).reshape(-1,1)\r\n",
        "        #입력된 원본 테스트 데이터를 폴드 세트내 학습된 기반 모델에서 예측 후 데이터 저장. \r\n",
        "        test_pred[:, folder_counter] = model.predict(X_test_n)\r\n",
        "\r\n",
        "        print('ok')\r\n",
        "            \r\n",
        "    # 폴드 세트 내에서 원본 테스트 데이터를 예측한 데이터를 평균하여 테스트 데이터로 생성 \r\n",
        "    test_pred_mean = np.mean(test_pred, axis=1).reshape(-1,1)    \r\n",
        "    \r\n",
        "    #train_fold_pred는 최종 메타 모델이 사용하는 학습 데이터, test_pred_mean은 테스트 데이터\r\n",
        "    return train_fold_pred , test_pred_mean"
      ],
      "execution_count": 256,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eh2ucxR3lHmX",
        "outputId": "10f1222b-9f3e-4f13-e320-8d711616b011"
      },
      "source": [
        "tst_x = [X_train[:,:max_len],X_train[:,-feature_num:]]\r\n",
        "tst_x"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([[0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 1.2000e+01, 1.4530e+03,\n",
              "         1.4000e+01],\n",
              "        [1.1690e+03, 1.3300e+02, 1.7510e+03, ..., 9.9710e+03, 1.8000e+01,\n",
              "         7.7400e+02],\n",
              "        [1.4800e+03, 6.1900e+02, 4.1000e+01, ..., 3.0500e+02, 8.4300e+03,\n",
              "         9.0000e+00],\n",
              "        ...,\n",
              "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 2.0000e+01, 7.4100e+02,\n",
              "         5.0000e+00],\n",
              "        [6.7400e+02, 1.1195e+04, 5.1840e+03, ..., 2.0000e+00, 7.9200e+02,\n",
              "         9.0900e+02],\n",
              "        [2.3200e+02, 1.1740e+03, 3.2920e+03, ..., 2.4000e+01, 1.9430e+03,\n",
              "         2.5640e+03]]),\n",
              " array([[0.        , 0.        , 0.33333333, 1.        ],\n",
              "        [0.        , 0.        , 0.44444444, 0.        ],\n",
              "        [0.        , 0.        , 0.7016129 , 0.        ],\n",
              "        ...,\n",
              "        [0.        , 0.        , 0.55357143, 0.        ],\n",
              "        [0.        , 0.        , 0.14705882, 0.        ],\n",
              "        [0.        , 0.        , 0.15789474, 0.        ]])]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 152
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "id": "DDz2_OZqtdRQ",
        "outputId": "8554d07f-a3c9-42a7-a89d-bcb67a9bcec4"
      },
      "source": [
        "\r\n",
        "# gb, ada, cb, ext, lstm \r\n",
        "\r\n",
        "# gb_train, gb_test = get_stacking_base_datasets(gb_grid_tst, train_X, train_y, test_X, 3)\r\n",
        "# ada_train, ada_test = get_stacking_base_datasets(ada_grid_tst, train_X, train_y, test_X, 3)\r\n",
        "cb_train, cb_test = get_stacking_base_datasets(cb_clf_tst, train_X, train_y, test_X, 3)\r\n",
        "# ext_train, ext_test = get_stacking_base_datasets(ext_cv_tst, train_X, train_y, test_X, 3) \r\n",
        "# lstm_train, lstm_test = get_stacking_base_datasets(lstm_tst_model, train_X, train_y, test_X, 3) \r\n"
      ],
      "execution_count": 260,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CatBoostClassifier  model 시작 \n",
            "\t 폴드 세트:  0  시작 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-260-9ab008cbfa3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# gb_train, gb_test = get_stacking_base_datasets(gb_grid_tst, train_X, train_y, test_X, 3)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# ada_train, ada_test = get_stacking_base_datasets(ada_grid_tst, train_X, train_y, test_X, 3)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mcb_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_stacking_base_datasets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcb_clf_tst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;31m# ext_train, ext_test = get_stacking_base_datasets(ext_cv_tst, train_X, train_y, test_X, 3)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m# lstm_train, lstm_test = get_stacking_base_datasets(lstm_tst_model, train_X, train_y, test_X, 3)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-256-1e67bbf682f5>\u001b[0m in \u001b[0;36mget_stacking_base_datasets\u001b[0;34m(model, X_train_n, y_train_n, X_test_n, n_folds)\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\t 폴드 세트: '\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfolder_counter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m' 시작 '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mX_tr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_train_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0my_tr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_train_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m         \u001b[0mX_te\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_train_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvalid_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    904\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    905\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 906\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    907\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    908\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m_get_with\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    939\u001b[0m             \u001b[0;31m#  (i.e. self.iloc) or label-based (i.e. self.loc)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    940\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_fallback_to_positional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 941\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    942\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    943\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    877\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    878\u001b[0m             \u001b[0mmaybe_callable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 879\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_callable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    880\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    881\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_is_scalar_access\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_getitem_axis\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1097\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot index with multidimensional key\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1098\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1099\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_iterable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1101\u001b[0m             \u001b[0;31m# nested tuple slicing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_getitem_iterable\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m         \u001b[0;31m# A collection of keys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1037\u001b[0;31m         \u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_listlike_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraise_missing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1038\u001b[0m         return self.obj._reindex_with_indexers(\n\u001b[1;32m   1039\u001b[0m             \u001b[0;34m{\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_dups\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_get_listlike_indexer\u001b[0;34m(self, key, axis, raise_missing)\u001b[0m\n\u001b[1;32m   1252\u001b[0m             \u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_indexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reindex_non_unique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1254\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_read_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraise_missing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mraise_missing\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1255\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_validate_read_indexer\u001b[0;34m(self, key, indexer, axis, raise_missing)\u001b[0m\n\u001b[1;32m   1314\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0moption_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"display.max_seq_items\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"display.width\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m80\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m                     raise KeyError(\n\u001b[0;32m-> 1316\u001b[0;31m                         \u001b[0;34m\"Passing list-likes to .loc or [] with any missing labels \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1317\u001b[0m                         \u001b[0;34m\"is no longer supported. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m                         \u001b[0;34mf\"The following labels were missing: {not_found}. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: \"Passing list-likes to .loc or [] with any missing labels is no longer supported. The following labels were missing: Int64Index([ 4576,  8761,  8765,  8766,  9335,\\n            ...\\n            11880, 11882, 11884, 11886, 17858],\\n           dtype='int64', length=43). See https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#deprecate-loc-reindex-listlike\""
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "0fH9U-Y1tdVz",
        "outputId": "5a0b8318-a513-4adf-87a7-78876e09f78a"
      },
      "source": [
        "Stack_final_X_train = np.concatenate((knn_train, rf_train, dt_train, ada_train), axis=1)\r\n",
        "Stack_final_X_test = np.concatenate((knn_test, rf_test, dt_test, ada_test), axis=1)\r\n",
        "\r\n",
        "print('원본 학습 피처 데이터 Shape:',X_train.shape, '원본 테스트 피처 Shape:',X_test.shape)\r\n",
        "print('스태킹 학습 피처 데이터 Shape:', Stack_final_X_train.shape,\r\n",
        "      '스태킹 테스트 피처 데이터 Shape:',Stack_final_X_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-127-7894a1ca730d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mStack_final_X_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mknn_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrf_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdt_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mada_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mStack_final_X_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mknn_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrf_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdt_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mada_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'원본 학습 피처 데이터 Shape:'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'원본 테스트 피처 Shape:'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m print('스태킹 학습 피처 데이터 Shape:', Stack_final_X_train.shape,\n",
            "\u001b[0;31mNameError\u001b[0m: name 'knn_train' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fVRfU68s5mFu"
      },
      "source": [
        "stack_final=LGBMClassifier(random_state=42)\r\n",
        "stack_final.fit(Stack_final_X_train, y_train)\r\n",
        "\r\n",
        "\r\n",
        "joblib.dump(stack_final,'stack_final_lgbm_clf.pkl')\r\n",
        "\r\n",
        "stack_predict = stack_final.predict(Stack_final_X_test)\r\n",
        "stack_predict"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sXTGqKXSGNnz"
      },
      "source": [
        "# submmit\r\n",
        "\r\n",
        "submission = pd.read_csv(\"/content/drive/MyDrive/Colab Notebooks/dacon_news/data/sample_submission.csv\")\r\n",
        "submission[\"info\"]= stack_predict\r\n",
        "\r\n",
        "submission.to_csv(\"stacking1_submission.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e5T3YCiNO2GP",
        "outputId": "4b1ae999-8308-4a7d-e5e9-5540ed2c8abd"
      },
      "source": [
        "test_X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565, 24)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 285
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HUHTB6MePrkC",
        "outputId": "155d4b1d-2de1-46db-b97b-b57c29be1dc7"
      },
      "source": [
        "train_X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(118676, 24)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 297
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        },
        "id": "kwf3CQ4oO4wP",
        "outputId": "dce078fe-38ce-464a-a96a-22266097da90"
      },
      "source": [
        "a = np.array([test_X[:,:max_len],test_X[:,-feature_num:]])  #.to_array()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-295-6b67335ad321>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_X\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_X\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mfeature_num\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m#.to_array()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m: could not broadcast input array from shape (142565,20) into shape (142565)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OXZo7_19QpZy",
        "outputId": "b6f09d74-9b65-42f7-88e2-e457dfe8edc2"
      },
      "source": [
        "test_X[:,-feature_num:max_len]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([], shape=(142565, 0), dtype=float64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 296
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p1rs5o0YPQYF",
        "outputId": "afbfa6cc-cd3a-45d3-f5f7-83c524e40746"
      },
      "source": [
        ".shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(142565, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 277
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QTGrQTG0_DcA"
      },
      "source": [
        "# 최종 훈련"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "euiZSXmb-p3h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d7d36dfd-544e-490d-a72f-42a0c0425f1f"
      },
      "source": [
        "# 실제 예측시\r\n",
        "index_seq = (train_X.argmax(axis=1)).argsort(axis=0)\r\n",
        "train_X = train_X[index_seq]\r\n",
        "train_y = train_y.iloc[index_seq]\r\n",
        "\r\n",
        "\r\n",
        "tf.random.set_seed(40)\r\n",
        "\r\n",
        "# 하이퍼파라미터\r\n",
        "max_epoch = 2\r\n",
        "batch_size = 32\r\n",
        "learning_rate = 0.001\r\n",
        "\r\n",
        "model = LSTM_add_feature(vocab_size, max_len = max_len)\r\n",
        "history = model.fit(x=[train_X[:,:-feature_num],train_X[:,-feature_num:]], y=train_y,epochs=max_epoch,batch_size = batch_size)\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_12\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_32 (InputLayer)           [(None, 20)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_12 (Embedding)        (None, 20, 200)      6692200     input_32[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout1d_12 (SpatialDr (None, 20, 200)      0           embedding_12[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "lstm_24 (LSTM)                  (None, 20, 32)       29824       spatial_dropout1d_12[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "lstm_25 (LSTM)                  (None, 32)           8320        lstm_24[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_12 (Dropout)            (None, 32)           0           lstm_25[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "input_33 (InputLayer)           [(None, 4)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_12 (Concatenate)    (None, 36)           0           dropout_12[0][0]                 \n",
            "                                                                 input_33[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense_55 (Dense)                (None, 32)           1184        concatenate_12[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "dense_56 (Dense)                (None, 1)            33          dense_55[0][0]                   \n",
            "==================================================================================================\n",
            "Total params: 6,731,561\n",
            "Trainable params: 6,731,561\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Epoch 1/2\n",
            "3701/3701 [==============================] - 349s 93ms/step - loss: 0.1053 - accuracy: 0.9589\n",
            "Epoch 2/2\n",
            "3701/3701 [==============================] - 348s 94ms/step - loss: 0.0238 - accuracy: 0.9926\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "laEwou3JphOQ"
      },
      "source": [
        "model.save(\"final_model_99.2.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0AYUPwxlrcI1"
      },
      "source": [
        "final_model = keras.models.load_model(\"final_model_99.2.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfCyi7aR2kDS"
      },
      "source": [
        "train_predicted = final_model.predict((train_X[:,:max_len],train_X[:,-feature_num:]))\r\n",
        "\r\n",
        "# 각 문장의 바로 앞문장과, 그 앞 문장의 예측값 생성\r\n",
        "train_pre_predicted1 = np.array([train_predicted[idx-1][0] for idx in range(len(train_predicted))]).reshape(-1,1) # 앞문장\r\n",
        "train_pre_predicted2 = np.array([train_predicted[idx-2][0] for idx in range(len(train_predicted))]).reshape(-1,1) # 앞앞문장\r\n",
        "train_pre_predicted3 = np.array([train_predicted[idx-3][0] for idx in range(len(train_predicted))]).reshape(-1,1) # 앞앞앞문장"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EI32y8JjAVEN"
      },
      "source": [
        "# ord가 1이면 과거 1년도, 2년도 자기값으로 대체\r\n",
        "# ord가 2이면 과거 2년도 자기값으로 대체\r\n",
        "\r\n",
        "train[\"predicted\"]=train_predicted\r\n",
        "train[\"pre_predicted1\"]=train_pre_predicted1\r\n",
        "train[\"pre_predicted2\"]=train_pre_predicted2\r\n",
        "\r\n",
        "train.loc[train[\"ord\"]==1,\"pre_predicted1\"]=train_predicted[train[\"ord\"]==1]\r\n",
        "train.loc[train[\"ord\"]==1,\"pre_predicted2\"]=train_predicted[train[\"ord\"]==1]\r\n",
        "train.loc[train[\"ord\"]==2,\"pre_predicted2\"]=train_predicted[train[\"ord\"]==2]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "to4KDsxX0lbj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c9cb9abd-9f82-4945-9518-08da76b68386"
      },
      "source": [
        "# tree 모델 불러오기\r\n",
        "import pickle\r\n",
        "import xgboost as xgb\r\n",
        "from xgboost import XGBClassifier\r\n",
        "from lightgbm import LGBMClassifier\r\n",
        "\r\n",
        "# xgb model 예측\r\n",
        "xgb_model=XGBClassifier(random_state=42)\r\n",
        "lgbm_model=LGBMClassifier(random_state=42)\r\n",
        "\r\n",
        "with open(path+'model/new_xgb_model.pkl', 'wb') as f:\r\n",
        "  pickle.dump(xgb_model, f)\r\n",
        "with open(path+'model/lgbm_model.pkl','wb') as f:\r\n",
        "  pickle.dump(lgbm_model,f)\r\n",
        "\r\n",
        "xgb_model.fit(train[[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]], y=train[\"info\"])\r\n",
        "lgbm_model.fit(train[[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]], y=train[\"info\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
              "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
              "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
              "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
              "               random_state=42, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
              "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 241
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKXg-UbCp3DX"
      },
      "source": [
        "def Lag_MLP():\r\n",
        "  model = keras.models.Sequential([\r\n",
        "    keras.layers.Dense(16, input_shape = (3,)),\r\n",
        "    keras.layers.Dense(1)                \r\n",
        "  ])\r\n",
        "  model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate), loss=\"binary_crossentropy\", metrics = \"accuracy\")\r\n",
        "  model.summary()\r\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XG5Zx2ymt_hm"
      },
      "source": [
        "# 시퀀셜 LSTM 모델\r\n",
        "\r\n",
        "def Lag_RNN():\r\n",
        "    model = keras.models.Sequential()\r\n",
        "    model.add(keras.layers.Input(shape=(3,1)))\r\n",
        "    model.add(keras.layers.SimpleRNN(32))\r\n",
        "    model.add(keras.layers.Dense(1, activation='sigmoid'))\r\n",
        "    model.compile(optimizer='adam', loss='binary_crossentropy', metrics='accuracy')\r\n",
        "    model.summary()\r\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qqeDPwzPuFkl",
        "outputId": "649d3e72-068e-4df6-a0e6-33a2af366ad4"
      },
      "source": [
        "# 하이퍼파라미터\r\n",
        "tf.random.set_seed(42)\r\n",
        "\r\n",
        "max_epoch = 5\r\n",
        "batch_size = 32\r\n",
        "learning_rate = 0.001\r\n",
        "\r\n",
        "lag_model = Lag_RNN()\r\n",
        "lag_model.fit(x=train[:82858][[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]].values.reshape(-1,3,1), y=train[:82858][\"info\"],\r\n",
        "              validation_data = (train[82858:][[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]].values.reshape(-1,3,1),train[82858:][\"info\"]), batch_size = 16,\r\n",
        "              epochs = max_epoch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_22\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "simple_rnn_7 (SimpleRNN)     (None, 32)                1088      \n",
            "_________________________________________________________________\n",
            "dense_31 (Dense)             (None, 1)                 33        \n",
            "=================================================================\n",
            "Total params: 1,121\n",
            "Trainable params: 1,121\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "5179/5179 [==============================] - 12s 2ms/step - loss: 0.0471 - accuracy: 0.9914 - val_loss: 0.0027 - val_accuracy: 0.9997\n",
            "Epoch 2/5\n",
            "5179/5179 [==============================] - 12s 2ms/step - loss: 0.0140 - accuracy: 0.9968 - val_loss: 0.0021 - val_accuracy: 0.9996\n",
            "Epoch 3/5\n",
            "5179/5179 [==============================] - 12s 2ms/step - loss: 0.0112 - accuracy: 0.9977 - val_loss: 0.0019 - val_accuracy: 0.9996\n",
            "Epoch 4/5\n",
            "5179/5179 [==============================] - 12s 2ms/step - loss: 0.0141 - accuracy: 0.9968 - val_loss: 0.0028 - val_accuracy: 0.9997\n",
            "Epoch 5/5\n",
            "5179/5179 [==============================] - 12s 2ms/step - loss: 0.0124 - accuracy: 0.9972 - val_loss: 0.0019 - val_accuracy: 0.9997\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f39e0bf3198>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OVKLgFtHqpbx",
        "outputId": "cad04740-ac7b-4d77-95e4-8b1054e05910"
      },
      "source": [
        "# 하이퍼파라미터\r\n",
        "tf.random.set_seed(42)\r\n",
        "\r\n",
        "max_epoch = 5\r\n",
        "batch_size = 32\r\n",
        "learning_rate = 0.0001\r\n",
        "\r\n",
        "lag_model = Lag_MLP()\r\n",
        "lag_model.fit(x=train[:82858][[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]], y=train[:82858][\"info\"],\r\n",
        "              validation_data = (train[82858:][[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]],train[82858:][\"info\"]), batch_size = batch_size,\r\n",
        "              epochs = max_epoch, validation_batch_size = batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_20\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_28 (Dense)             (None, 16)                64        \n",
            "_________________________________________________________________\n",
            "dense_29 (Dense)             (None, 1)                 17        \n",
            "=================================================================\n",
            "Total params: 81\n",
            "Trainable params: 81\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "2590/2590 [==============================] - 4s 1ms/step - loss: 0.1080 - accuracy: 0.9517 - val_loss: 0.0027 - val_accuracy: 0.9995\n",
            "Epoch 2/5\n",
            "2590/2590 [==============================] - 4s 1ms/step - loss: 0.0168 - accuracy: 0.9961 - val_loss: 0.0019 - val_accuracy: 0.9995\n",
            "Epoch 3/5\n",
            "2590/2590 [==============================] - 4s 1ms/step - loss: 0.0132 - accuracy: 0.9968 - val_loss: 0.0017 - val_accuracy: 0.9995\n",
            "Epoch 4/5\n",
            "2590/2590 [==============================] - 4s 1ms/step - loss: 0.0212 - accuracy: 0.9962 - val_loss: 0.0029 - val_accuracy: 0.9995\n",
            "Epoch 5/5\n",
            "2590/2590 [==============================] - 4s 1ms/step - loss: 0.0169 - accuracy: 0.9966 - val_loss: 0.0029 - val_accuracy: 0.9995\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f39f2c0b0b8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LDjRKT6a_Tx3"
      },
      "source": [
        "# test 예측"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "efZxw2tUiMfr"
      },
      "source": [
        "path = \"/gdrive/My Drive/dacon_news/\"\r\n",
        "feature_num = 4\r\n",
        "test = pd.read_csv(path+\"data/news_test.csv\")\r\n",
        "\r\n",
        "# EDA 기반 feature 추가\r\n",
        "test[\"info1_title\"]=test[\"title\"].apply(title_choose)\r\n",
        "test[\"info1_content\"]=test[\"content\"].apply(content_choose)\r\n",
        "test[\"content_startswith_[\"]=test.content.apply(lambda x : str(x).startswith(\"[\" ) or str(x).startswith(\"(\") or str(x).startswith(\"제목\"))+0\r\n",
        "title_group = (test.groupby([\"title\"]).count())[\"n_id\"]\r\n",
        "test[\"new_ord\"]=test.apply(lambda x: x[\"ord\"]/title_group[x[\"title\"]], axis=1)\r\n",
        "#test[\"info1_words\"]=test[\"content\"].apply(in_bull)\r\n",
        "\r\n",
        "# 전처리\r\n",
        "test['new_article'], okt = text_preprocessing(test['content'])\r\n",
        "test_X_seq = vectorizer.texts_to_sequences(test[\"new_article\"])\r\n",
        "test_X = pad_sequences(test_X_seq, maxlen = max_len) # 길이를 맞춰줌\r\n",
        "\r\n",
        "# concat\r\n",
        "test_X = np.concatenate([test_X,test[[\"info1_title\",\"info1_content\",\"new_ord\",\"content_startswith_[\"]].values.reshape(-1,feature_num)], axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wtfXQsVKkuAQ"
      },
      "source": [
        "#final_model = keras.models.load_model(path+\"model/final_model_99_epoch2.h5\") # 저장된 모델 불러오기\r\n",
        "final_model = keras.models.load_model(\"final_model_99.2.h5\")\r\n",
        "predicted = final_model.predict([test_X[:,:max_len],test_X[:,-feature_num:]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sTwBLw2sN2T"
      },
      "source": [
        "d = "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ncycsgEs4jH"
      },
      "source": [
        "# idx-1의 예측값\r\n",
        "pre_predicted1 = np.array([predicted[idx-1][0] for idx in range(len(predicted))]).reshape(-1,1)\r\n",
        "pre_predicted2 = np.array([predicted[idx-2][0] for idx in range(len(predicted))]).reshape(-1,1)\r\n",
        "\r\n",
        "test[\"predicted\"]=predicted\r\n",
        "test[\"pre_predicted1\"]=pre_predicted1\r\n",
        "test[\"pre_predicted2\"]=pre_predicted2\r\n",
        "\r\n",
        "# ord가 1이면 과거 1년도, 2년도 자기값으로 대체\r\n",
        "# ord가 2이면 과거 2년도 자기값으로 대체\r\n",
        "test.loc[test[\"ord\"]==1,\"pre_predicted1\"]=predicted[test[\"ord\"]==1]\r\n",
        "test.loc[test[\"ord\"]==1,\"pre_predicted2\"]=predicted[test[\"ord\"]==1]\r\n",
        "test.loc[test[\"ord\"]==2,\"pre_predicted2\"]=predicted[test[\"ord\"]==2]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "niuyb-3nrKxT"
      },
      "source": [
        "xgb_final_predicted = xgb_model.predict(test[[\"predicted\",\"pre_predicted1\"]])\r\n",
        "lgbm_final_predicted = lgbm_model.predict(test[[\"predicted\",\"pre_predicted1\"]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G8Wg-C6duF37"
      },
      "source": [
        "xgb_final_predicted = xgb_model.predict(test[[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]])\r\n",
        "lgbm_final_predicted = lgbm_model.predict(test[[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eH86eXK-1voh"
      },
      "source": [
        "wrong = test[(xgb_final_predicted)!=(lgbm_final_predicted)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xRR4v_vR4Nx3",
        "outputId": "9a55a515-c912-438d-b3ff-fdec61d72a3e"
      },
      "source": [
        "wrong[\"xgb_predicted\"]=xgb_final_predicted[xgb_final_predicted!=lgbm_final_predicted]\r\n",
        "wrong[\"lgbm_predicte\"]=lgbm_final_predicted[xgb_final_predicted!=lgbm_final_predicted]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8pDAbLAS6Dn5"
      },
      "source": [
        "xgb_train_p = xgb_model.predict(train[[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]])\r\n",
        "lgbm_train_p = lgbm_model.predict(train[[\"predicted\",\"pre_predicted1\",\"pre_predicted2\"]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tgw368iD8jWV",
        "outputId": "967da57e-d3e6-4eea-8031-60ccff70d2ef"
      },
      "source": [
        "wrong_train = train[(xgb_train_p)!=(lgbm_train_p)]\r\n",
        "\r\n",
        "wrong_train[\"xgb_predicted\"]=xgb_train_p[xgb_train_p!=lgbm_train_p]\r\n",
        "wrong_train[\"lgbm_predicted\"]=lgbm_train_p[xgb_train_p!=lgbm_train_p]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2eyxDj1Gts9k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27185148-9cea-4366-f2d9-ec85db102417"
      },
      "source": [
        "test[\"info\"]=xgb_final_predicted\r\n",
        "#test[\"info\"][test[\"content\"].apply(lambda x: True if \"무단전재 & 재배포 금지\" in x else False)]=0\r\n",
        "test[\"info\"][test[\"content\"].apply(lambda x: True if ('http://etoday.bujane.co.kr/' in x) or ('http://bit.ly/2XrAuGJ_itoozanews' in x) or ('http://www.hisl.co.kr/0306/' in x) or ('https://www.hankyung.com/election2020/' in x) or (x==']]') else False)]=1\r\n",
        "#test[\"info\"][test[\"predicted\"]>=0.6]=1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  after removing the cwd from sys.path.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BygcXFgCDoPl"
      },
      "source": [
        "submission = pd.read_csv(path+\"/data/sample_submission.csv\")\r\n",
        "submission[\"info\"]=test[\"info\"]\r\n",
        "submission.to_csv(\"xgb_drop0.5_submission.csv\", encoding=\"utf-8-sig\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 407
        },
        "id": "hj-gsYUpOgOR",
        "outputId": "e0911b94-1483-4856-dc14-4f38aeece5f8"
      },
      "source": [
        "submission"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>info</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NEWS00237_1</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NEWS00237_2</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NEWS00237_3</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NEWS00237_4</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NEWS00237_5</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142560</th>\n",
              "      <td>NEWS09482_72</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142561</th>\n",
              "      <td>NEWS09482_73</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142562</th>\n",
              "      <td>NEWS09482_74</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142563</th>\n",
              "      <td>NEWS09482_75</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142564</th>\n",
              "      <td>NEWS09482_76</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>142565 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                  id  info\n",
              "0        NEWS00237_1   0.0\n",
              "1        NEWS00237_2   0.0\n",
              "2        NEWS00237_3   0.0\n",
              "3        NEWS00237_4   0.0\n",
              "4        NEWS00237_5   0.0\n",
              "...              ...   ...\n",
              "142560  NEWS09482_72   1.0\n",
              "142561  NEWS09482_73   1.0\n",
              "142562  NEWS09482_74   1.0\n",
              "142563  NEWS09482_75   1.0\n",
              "142564  NEWS09482_76   1.0\n",
              "\n",
              "[142565 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 122
        }
      ]
    }
  ]
}