textsum_seq2seq_ver2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.6.0-dev20180117\n"
     ]
    }
   ],
   "source": [
    "from distutils.version import LooseVersion\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "\n",
    "\n",
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.1'), 'Please use TensorFlow version 1.1 or newer'\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import tensorflow as tf\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_path='./vocab/vocab.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_vocab(vocab_path):\n",
    "    vocab_list=[]\n",
    "    with open(vocab_path,'r',encoding='utf-8')as f:\n",
    "        for item in f.readlines():\n",
    "            vocab_list.append(item.strip())\n",
    "    int_to_vocab = {idx: word for idx, word in enumerate(vocab_list)}\n",
    "    vocab_to_int = {word: idx for idx, word in int_to_vocab.items()}\n",
    "    return int_to_vocab,vocab_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_int_to_letter, source_letter_to_int=get_vocab(vocab_path)\n",
    "target_int_to_letter, target_letter_to_int =source_int_to_letter, source_letter_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(source_int_to_letter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  构建模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 输入层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_inputs():\n",
    "    '''\n",
    "    模型输入tensor\n",
    "    '''\n",
    "    inputs = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    learning_rate = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    \n",
    "    # 定义target序列最大长度（之后target_sequence_length和source_sequence_length会作为feed_dict的参数）\n",
    "    target_sequence_length = tf.placeholder(tf.int32, (None,), name='target_sequence_length')\n",
    "    max_target_sequence_length = tf.reduce_max(target_sequence_length, name='max_target_len')\n",
    "    source_sequence_length = tf.placeholder(tf.int32, (None,), name='source_sequence_length')\n",
    "    \n",
    "    return inputs, targets, learning_rate, target_sequence_length, max_target_sequence_length, source_sequence_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_encoder_layer(input_data, rnn_size, num_layers,\n",
    "                   source_sequence_length, source_vocab_size, \n",
    "                   encoding_embedding_size):\n",
    "\n",
    "    '''\n",
    "    构造Encoder层\n",
    "    \n",
    "    参数说明：\n",
    "    - input_data: 输入tensor\n",
    "    - rnn_size: rnn隐层结点数量\n",
    "    - num_layers: 堆叠的rnn cell数量\n",
    "    - source_sequence_length: 源数据的序列长度\n",
    "    - source_vocab_size: 源数据的词典大小\n",
    "    - encoding_embedding_size: embedding的大小\n",
    "    '''\n",
    "    # Encoder embedding\n",
    "    encoder_embed_input = tf.contrib.layers.embed_sequence(input_data, source_vocab_size, encoding_embedding_size)\n",
    "\n",
    "    # RNN cell\n",
    "    def get_lstm_cell(rnn_size):\n",
    "        lstm_cell = tf.contrib.rnn.LSTMCell(rnn_size, initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))\n",
    "        return lstm_cell\n",
    "\n",
    "    cell = tf.contrib.rnn.MultiRNNCell([get_lstm_cell(rnn_size) for _ in range(num_layers)])\n",
    "    \n",
    "    encoder_output, encoder_state = tf.nn.dynamic_rnn(cell, encoder_embed_input, \n",
    "                                                      sequence_length=source_sequence_length, dtype=tf.float32)\n",
    "    \n",
    "    return encoder_output, encoder_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_decoder_input(data, vocab_to_int, batch_size):\n",
    "    '''\n",
    "    补充<GO>，并移除最后一个字符\n",
    "    \n",
    "    这里是为了构建 Decoder训练时的输入数据，使用target 而不是预测出的数据，提高精度\n",
    "    '''\n",
    "    # cut掉最后一个字符\n",
    "    ending = tf.strided_slice(data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    decoder_input = tf.concat([tf.fill([batch_size, 1], vocab_to_int['<GO>']), ending], 1)\n",
    "\n",
    "    return decoder_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decoding_layer(target_letter_to_int, decoding_embedding_size, num_layers, rnn_size,\n",
    "                   target_sequence_length, max_target_sequence_length, encoder_state, decoder_input,\n",
    "                   encoder_outputs,source_sequence_length):\n",
    "    '''\n",
    "    构造Decoder层\n",
    "    \n",
    "    参数：\n",
    "    - target_letter_to_int: target数据的映射表\n",
    "    - decoding_embedding_size: embed向量大小\n",
    "    - num_layers: 堆叠的RNN单元数量\n",
    "    - rnn_size: RNN单元的隐层结点数量\n",
    "    - target_sequence_length: target数据序列长度\n",
    "    - max_target_sequence_length: target数据序列最大长度\n",
    "    - encoder_state: encoder端编码的状态向量\n",
    "    - decoder_input: decoder端输入\n",
    "    \n",
    "    - encoder_outputs :添加一个注意力机制\n",
    "    - source_sequence_length 源数据长度\n",
    "    '''\n",
    "    \n",
    "    '''\n",
    "    # attention_states: [batch_size, max_time, num_units]\n",
    "    attention_states = tf.transpose(encoder_outputs, [1, 0, 2])\n",
    "\n",
    "    # Create an attention mechanism\n",
    "    attention_mechanism = tf.contrib.seq2seq.LuongAttention(\n",
    "        num_units, attention_states,\n",
    "        memory_sequence_length=source_sequence_length)\n",
    "    \n",
    "    decoder_cell = tf.contrib.seq2seq.AttentionWrapper(\n",
    "    decoder_cell, attention_mechanism,\n",
    "    attention_layer_size=num_units)\n",
    "    \n",
    "    \n",
    "    '''\n",
    "    # 1. Embedding\n",
    "    target_vocab_size = len(target_letter_to_int)\n",
    "    decoder_embeddings = tf.Variable(tf.random_uniform([target_vocab_size, decoding_embedding_size]))\n",
    "    decoder_embed_input = tf.nn.embedding_lookup(decoder_embeddings, decoder_input)\n",
    "\n",
    "    # 2. 构造Decoder中的RNN单元\n",
    "    def get_decoder_cell(rnn_size):\n",
    "        decoder_cell = tf.contrib.rnn.LSTMCell(rnn_size,\n",
    "                                           initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))\n",
    "        return decoder_cell\n",
    "    \n",
    "    #2.1 添加注意力机制的RNN 单元\n",
    "    def get_decoder_cell_attention(rnn_size):\n",
    "         # attention_states: [batch_size, max_time, num_units]\n",
    "#        attention_states = tf.transpose(encoder_outputs, [1, 0, 2])\n",
    "        attention_states=encoder_outputs\n",
    "         # Create an attention mechanism\n",
    "        attention_mechanism = tf.contrib.seq2seq.LuongAttention(\n",
    "        rnn_size, attention_states,\n",
    "        memory_sequence_length=source_sequence_length)\n",
    "        \n",
    "        decoder_cell = tf.contrib.rnn.LSTMCell(rnn_size,\n",
    "                                           initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))\n",
    "        decoder_cell = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                            decoder_cell, attention_mechanism,\n",
    "                            attention_layer_size=rnn_size)\n",
    "        \n",
    "        return decoder_cell\n",
    "    \n",
    "    \n",
    "#     cell = tf.contrib.rnn.MultiRNNCell([get_decoder_cell(rnn_size) for _ in range(num_layers)])\n",
    "    cell = tf.contrib.rnn.MultiRNNCell([get_decoder_cell_attention(rnn_size) for _ in range(num_layers)])\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    # 3. Output全连接层\n",
    "    output_layer = Dense(target_vocab_size,\n",
    "                         kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
    "\n",
    "\n",
    "    # 4. Training decoder\n",
    "    with tf.variable_scope(\"decode\"):\n",
    "        # 得到help对象\n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(inputs=decoder_embed_input,\n",
    "                                                            sequence_length=target_sequence_length,\n",
    "                                                            time_major=False)\n",
    "        # 构造decoder\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(cell,\n",
    "                                                           training_helper,\n",
    "                                                           initial_state=cell.zero_state(dtype=tf.float32,batch_size=batch_size)\n",
    "                                                        ,output_layer=output_layer) \n",
    "\n",
    "        training_decoder_output, _ ,_= tf.contrib.seq2seq.dynamic_decode(training_decoder,\n",
    "                                                                       impute_finished=True,\n",
    "                                                                       maximum_iterations=max_target_sequence_length)\n",
    "    # 5. Predicting decoder\n",
    "    # 与training共享参数\n",
    "    with tf.variable_scope(\"decode\", reuse=True):\n",
    "        # 创建一个常量tensor并复制为batch_size的大小\n",
    "        start_tokens = tf.tile(tf.constant([target_letter_to_int['<GO>']], dtype=tf.int32), [batch_size], \n",
    "                               name='start_tokens')\n",
    "        predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(decoder_embeddings,\n",
    "                                                                start_tokens,\n",
    "                                                                target_letter_to_int['<EOS>'])\n",
    "        predicting_decoder = tf.contrib.seq2seq.BasicDecoder(cell,\n",
    "                                                        predicting_helper,\n",
    "                                                        initial_state=cell.zero_state(dtype=tf.float32,batch_size=batch_size)\n",
    "                                                            ,output_layer=output_layer)\n",
    "        predicting_decoder_output, _,_  = tf.contrib.seq2seq.dynamic_decode(predicting_decoder,\n",
    "                                                            impute_finished=True,\n",
    "                                                            maximum_iterations=max_target_sequence_length)\n",
    "    \n",
    "    return training_decoder_output, predicting_decoder_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## seq2seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seq2seq_model(input_data, targets, lr, target_sequence_length, \n",
    "                  max_target_sequence_length, source_sequence_length,\n",
    "                  source_vocab_size, target_vocab_size,\n",
    "                  encoder_embedding_size, decoder_embedding_size, \n",
    "                  rnn_size, num_layers):\n",
    "    \n",
    "    # 获取encoder的状态输出\n",
    "    encoder_outputs, encoder_state = get_encoder_layer(input_data, \n",
    "                                  rnn_size, \n",
    "                                  num_layers, \n",
    "                                  source_sequence_length,\n",
    "                                  source_vocab_size, \n",
    "                                  encoding_embedding_size)\n",
    "    \n",
    "    \n",
    "    # 预处理后的decoder输入\n",
    "    decoder_input = process_decoder_input(targets, target_letter_to_int, batch_size)\n",
    "    \n",
    "    # 将状态向量与输入传递给decoder\n",
    "    training_decoder_output, predicting_decoder_output = decoding_layer(target_letter_to_int, \n",
    "                                                                       decoding_embedding_size, \n",
    "                                                                       num_layers, \n",
    "                                                                       rnn_size,\n",
    "                                                                       target_sequence_length,\n",
    "                                                                       max_target_sequence_length,\n",
    "                                                                       encoder_state, \n",
    "                                                                       decoder_input,\n",
    "                                                                        encoder_outputs,\n",
    "                                                                        source_sequence_length\n",
    "                                                                       ) \n",
    "    \n",
    "    return training_decoder_output, predicting_decoder_output\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 超参数\n",
    "# Number of Epochs\n",
    "epochs =50\n",
    "# Batch Size\n",
    "batch_size =32\n",
    "# RNN Size\n",
    "rnn_size = 50\n",
    "# Number of Layers\n",
    "num_layers = 2\n",
    "# Embedding Size\n",
    "encoding_embedding_size = 128\n",
    "decoding_embedding_size = 128\n",
    "# Learning Rate\n",
    "learning_rate = 0.05"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构造graph\n",
    "train_graph = tf.Graph()\n",
    "\n",
    "with train_graph.as_default():\n",
    "    \n",
    "    # 获得模型输入    \n",
    "    input_data, targets, lr, target_sequence_length, max_target_sequence_length, source_sequence_length = get_inputs()\n",
    "    \n",
    "    training_decoder_output, predicting_decoder_output = seq2seq_model(input_data, \n",
    "                                                                      targets, \n",
    "                                                                      lr, \n",
    "                                                                      target_sequence_length, \n",
    "                                                                      max_target_sequence_length, \n",
    "                                                                      source_sequence_length,\n",
    "                                                                      len(source_letter_to_int),\n",
    "                                                                      len(target_letter_to_int),\n",
    "                                                                      encoding_embedding_size, \n",
    "                                                                      decoding_embedding_size, \n",
    "                                                                      rnn_size, \n",
    "                                                                      num_layers)    \n",
    "    \n",
    "    training_logits = tf.identity(training_decoder_output.rnn_output, 'logits')\n",
    "    predicting_logits = tf.identity(predicting_decoder_output.sample_id, name='predictions')\n",
    "    \n",
    "    masks = tf.sequence_mask(target_sequence_length, max_target_sequence_length, dtype=tf.float32, name='masks')\n",
    "\n",
    "    with tf.name_scope(\"optimization\"):\n",
    "        \n",
    "        # Loss function\n",
    "        cost = tf.contrib.seq2seq.sequence_loss(\n",
    "            training_logits,\n",
    "            targets,\n",
    "            masks)\n",
    "\n",
    "        # Optimizer\n",
    "        optimizer = tf.train.AdamOptimizer(lr)\n",
    "\n",
    "        # Gradient Clipping\n",
    "        gradients = optimizer.compute_gradients(cost)\n",
    "        capped_gradients = [(tf.clip_by_value(grad, -5., 5.), var) for grad, var in gradients if grad is not None]\n",
    "        train_op = optimizer.apply_gradients(capped_gradients)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BasicDecoderOutput(rnn_output=<tf.Tensor 'decode/decoder/transpose:0' shape=(32, ?, 50000) dtype=float32>, sample_id=<tf.Tensor 'decode/decoder/transpose_1:0' shape=(32, ?) dtype=int32>)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_decoder_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    '''\n",
    "    对batch中的序列进行补全，保证batch中的每行都有相同的sequence_length\n",
    "    \n",
    "    参数：\n",
    "    - sentence batch\n",
    "    - pad_int: <PAD>对应索引号\n",
    "    '''\n",
    "    max_sentence = max([len(sentence) for sentence in sentence_batch])\n",
    "    return [sentence + [pad_int] * (max_sentence - len(sentence)) for sentence in sentence_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batches(file_list,tokenize_path,batch_size,pad_int):\n",
    "    '''\n",
    "    定义生成器，用来获取tokenize下的所有content\n",
    "    '''\n",
    "    for item in file_list:\n",
    "        source_path=os.path.join(tokenize_path,'content_'+item)\n",
    "        target_path=os.path.join(tokenize_path,'title_'+item)\n",
    "        with open(source_path,'r',encoding='utf-8')as sf:\n",
    "            sources=[[int(word) for word in sentence.strip().split(' ')]for sentence in sf.readlines()]\n",
    "        with open(target_path,'r',encoding='utf-8')as tf:\n",
    "            targets=[[int(word) for word in sentence.strip().split(' ')]for sentence in tf.readlines()]\n",
    "        \n",
    "        for batch_i in range(0, len(sources)//batch_size):\n",
    "            start_i = batch_i * batch_size\n",
    "            sources_batch = sources[start_i:start_i + batch_size]\n",
    "            targets_batch = targets[start_i:start_i + batch_size]\n",
    "            # 补全序列\n",
    "            pad_sources_batch = np.array(pad_sentence_batch(sources_batch, pad_int))\n",
    "            pad_targets_batch = np.array(pad_sentence_batch(targets_batch, pad_int))\n",
    "\n",
    "            # 记录每条记录的长度\n",
    "            targets_lengths = []\n",
    "            for target in targets_batch:\n",
    "                targets_lengths.append(len(target))\n",
    "\n",
    "            source_lengths = []\n",
    "            for source in sources_batch:\n",
    "                source_lengths.append(len(source))\n",
    "\n",
    "            yield pad_targets_batch, pad_sources_batch, targets_lengths, source_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path='./data_test'\n",
    "file_list=os.listdir(data_path)\n",
    "tokenize_path='./tokenize'\n",
    "batch_size=32\n",
    "pad_int=source_letter_to_int['<PAD>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "(valid_targets_batch, valid_sources_batch, valid_targets_lengths, valid_sources_lengths) = next(get_batches(file_list,tokenize_path,bath_size,pad_int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   1/50 Batch    0/未知 - Training Loss: 10.820  - Validation loss:  9.736\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-50-c7799fbf1baa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     19\u001b[0m                  \u001b[0mlr\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m                  \u001b[0mtarget_sequence_length\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtargets_lengths\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m                  source_sequence_length: sources_lengths})\n\u001b[0m\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mbatch_i\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mdisplay_step\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    893\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 895\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    896\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1126\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1127\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1128\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1129\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1130\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1342\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1343\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1344\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1345\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1346\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1348\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1349\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1350\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1351\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1352\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1327\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[0;32m   1328\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1329\u001b[1;33m                                    status, run_metadata)\n\u001b[0m\u001b[0;32m   1330\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1331\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "display_step = 50 # 每隔50轮输出loss\n",
    "\n",
    "# 查看是否有 checkpoint_f 这个文件，无则新建一个\n",
    "if not os.path.exists('./checkpoint_f'):\n",
    "        os.makedirs('./checkpoint_f')\n",
    "\n",
    "checkpoint = \"./checkpoint_f/trained_model.ckpt\" \n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "    for epoch_i in range(1, epochs+1):\n",
    "        for batch_i, (targets_batch, sources_batch, targets_lengths, sources_lengths) in enumerate(\n",
    "                get_batches(file_list,tokenize_path,batch_size,pad_int)):\n",
    "            \n",
    "            _, loss = sess.run(\n",
    "                [train_op, cost],\n",
    "                {input_data: sources_batch,\n",
    "                 targets: targets_batch,\n",
    "                 lr: learning_rate,\n",
    "                 target_sequence_length: targets_lengths,\n",
    "                 source_sequence_length: sources_lengths})\n",
    "\n",
    "            if batch_i % display_step == 0:\n",
    "            \n",
    "                # 计算validation loss\n",
    "                validation_loss = sess.run(\n",
    "                [cost],\n",
    "                {input_data: valid_sources_batch,\n",
    "                 targets: valid_targets_batch,\n",
    "                 lr: learning_rate,\n",
    "                 target_sequence_length: valid_targets_lengths,\n",
    "                 source_sequence_length: valid_sources_lengths})\n",
    "                \n",
    "                print('Epoch {:>3}/{} Batch {:>4}/{} - Training Loss: {:>6.3f}  - Validation loss: {:>6.3f}'\n",
    "                      .format(epoch_i,\n",
    "                              epochs, \n",
    "                              batch_i, \n",
    "                              '未知', \n",
    "                              loss, \n",
    "                              validation_loss[0]))\n",
    "\n",
    "    \n",
    "    \n",
    "    # 保存模型\n",
    "    saver = tf.train.Saver()\n",
    "    saver.save(sess, checkpoint)\n",
    "    print('Model Trained and Saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def source_to_seq(text):\n",
    "    '''\n",
    "    对源数据进行转换\n",
    "    '''\n",
    "    sequence_length = 120\n",
    "    return [source_letter_to_int.get(word, source_letter_to_int['<UNK>']) for word in text.split(' ')] + [source_letter_to_int['<PAD>']]*(sequence_length-len(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./checkpoint_f/trained_model.ckpt\n",
      "原始输入: \n",
      " NUMBER   反对 宪法 基本 原则 危害 国家 安全 政权 稳定 统一 的 煽动 民族 仇恨 民族 歧视 的   NUMBER   宣扬 邪教 和 封建迷信 的 散布 谣言 破坏 社会 稳定 的 侮辱 诽谤 他人 侵害 他人 合法权益 的   NUMBER   散布 淫秽 色情 赌博 暴力 凶杀 恐怖 或者 教唆 犯罪 的\n",
      " \n",
      "\n",
      "Source\n",
      "  Word 编号:    [1, 6, 4, 4, 710, 1299, 315, 738, 1237, 102, 262, 1419, 263, 592, 5, 1517, 423, 1504, 423, 1470, 5, 4, 4, 6, 4, 4, 1445, 1534, 10, 1545, 5, 727, 1353, 1010, 170, 263, 5, 1478, 1493, 618, 1316, 618, 1229, 5, 4, 4, 6, 4, 4, 727, 1525, 1434, 1275, 1225, 1546, 1363, 300, 1554, 1046, 1, 4]\n",
      "  Input Words: <UNK> NUMBER   反对 宪法 基本 原则 危害 国家 安全 政权 稳定 统一 的 煽动 民族 仇恨 民族 歧视 的   NUMBER   宣扬 邪教 和 封建迷信 的 散布 谣言 破坏 社会 稳定 的 侮辱 诽谤 他人 侵害 他人 合法权益 的   NUMBER   散布 淫秽 色情 赌博 暴力 凶杀 恐怖 或者 教唆 犯罪 <UNK> \n",
      "\n",
      "Target\n",
      "  Word 编号:       [71, 987, 3]\n",
      "  Response Words: 搜狐 房产 <EOS>\n"
     ]
    }
   ],
   "source": [
    "# 输入一个单词\n",
    "input_word = '''\n",
    " NUMBER   反对 宪法 基本 原则 危害 国家 安全 政权 稳定 统一 的 煽动 民族 仇恨 民族 歧视 的   NUMBER   宣扬 邪教 和 封建迷信 的 散布 谣言 破坏 社会 稳定 的 侮辱 诽谤 他人 侵害 他人 合法权益 的   NUMBER   散布 淫秽 色情 赌博 暴力 凶杀 恐怖 或者 教唆 犯罪 的\n",
    " '''\n",
    "text = source_to_seq(input_word)\n",
    "\n",
    "checkpoint = \"./checkpoint_f/trained_model.ckpt\"\n",
    "\n",
    "loaded_graph = tf.Graph()\n",
    "with tf.Session(graph=loaded_graph) as sess:\n",
    "    # 加载模型\n",
    "    loader = tf.train.import_meta_graph(checkpoint + '.meta')\n",
    "    loader.restore(sess, checkpoint)\n",
    "\n",
    "    input_data = loaded_graph.get_tensor_by_name('inputs:0')\n",
    "    logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
    "    source_sequence_length = loaded_graph.get_tensor_by_name('source_sequence_length:0')\n",
    "    target_sequence_length = loaded_graph.get_tensor_by_name('target_sequence_length:0')\n",
    "    \n",
    "    answer_logits = sess.run(logits, {input_data: [text]*batch_size, \n",
    "                                      target_sequence_length: [len(input_word)]*batch_size, \n",
    "                                      source_sequence_length: [len(input_word)]*batch_size})[0] \n",
    "\n",
    "\n",
    "pad = source_letter_to_int[\"<PAD>\"] \n",
    "\n",
    "print('原始输入:', input_word)\n",
    "\n",
    "print('\\nSource')\n",
    "print('  Word 编号:    {}'.format([i for i in text]))\n",
    "print('  Input Words: {}'.format(\" \".join([source_int_to_letter[i] for i in text])))\n",
    "\n",
    "print('\\nTarget')\n",
    "print('  Word 编号:       {}'.format([i for i in answer_logits if i != pad]))\n",
    "print('  Response Words: {}'.format(\" \".join([target_int_to_letter[i] for i in answer_logits if i != pad])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}