best.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "This is the version for search hyper-parameters by using the package Ray.\n",
    "you need install it with pip\n",
    "\"\"\"\n",
    "from operator import pos\n",
    "from re import A, S\n",
    "import os\n",
    "from typing import Dict, List\n",
    "from deepctr_torch.layers.interaction import BiInteractionPooling\n",
    "import numpy as np\n",
    "from numpy.core.defchararray import index\n",
    "from numpy.lib.type_check import real\n",
    "import pandas as pd\n",
    "from pandas.core.algorithms import isin\n",
    "import torch\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
    "from torch.nn.modules import linear\n",
    "from torch.nn.parameter import Parameter\n",
    "from tqdm import tqdm\n",
    "from deepctr_torch.inputs import SparseFeat, DenseFeat, get_feature_names\n",
    "from deepctr_torch.models.deepfm import *\n",
    "from deepctr_torch.models.basemodel import *\n",
    "from deepctr_torch.callbacks import EarlyStopping\n",
    "import time\n",
    "import argparse\n",
    "import math\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "from ray import tune\n",
    "from ray.tune import CLIReporter\n",
    "from ray.tune.schedulers import ASHAScheduler\n",
    "\n",
    "from utils import metrics\n",
    "from decfair_tune import FastFairNFM\n",
    "from baseline_tune import DeepNFM, NFM, MyDeepFM, MyFM, DeepFM, FM_witout_FirstOrder, FairNFM\n",
    "from counterfactual_reasoning import CR_NFM\n",
    "from pre_data_wechat import *\n",
    "from FairGo import myNFM,FairGo\n",
    "from ipw_nfm import ipw_myNFM\n",
    "from decfair_tune_nfm import DCR_NFM,DCR_NFM_apr\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]='0'\n",
    "# 存储数据的根目录\n",
    "ROOT_PATH = \"/home/zyang/code-2021/decFair/data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class parameters(object):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "        self.lr = 1e-2\n",
    "        self.reg_emb = 1e-3\n",
    "        self.reg_para = 1e-2\n",
    "        self.dim = 16\n",
    "        self.batch_size=1024\n",
    "        self.epoch = 100\n",
    "        self.use_videolen = 0\n",
    "        self.patience = 10\n",
    "        self.use_codelen = 1\n",
    "        self.model= 'fastFairNFM' # 'NFM' \n",
    "        self.post_action='like' #comment\n",
    "        self.stop_refer = 'val_do_ndcg_post10' \n",
    "        self.opt = 'adagrad'\n",
    "        self.pretrain = 0\n",
    "        self.alpha = 0.5\n",
    "        self.cuda=0\n",
    "        self.pretrained_model=\"/data/zyang/decFair/logs/best-kwai-adagrad-myNFM-vl-0-vc-1-like-myNFM-lr_0.01-reg_emb_0.001-reg_para_0-dim_16-stop-val_ndcg_post10-auxloss-1.0-stop-val_ndcg_post10-m.pth\"\n",
    "    def reset(self, config:Dict):\n",
    "        for name,val in config.items():\n",
    "            setattr(self,name,val)\n",
    "\n",
    "def change_columns_name(columns_name,uname='user_id',iname='item_id'):\n",
    "    index_u = columns_name.index(uname)\n",
    "    columns_name[index_u] = 'user_id'\n",
    "    index_i = columns_name.index(iname)\n",
    "    columns_name[index_i] = 'item_id'\n",
    "    return columns_name\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_a_model(config,need_train=False,model_path=None,mode='do'):\n",
    "    dataset='kwai'\n",
    "    print(\"dataset:\",dataset)\n",
    "    args = parameters()\n",
    "    args.reset(config)\n",
    "    used_optimizer = args.opt\n",
    "    lr = args.lr\n",
    "    post_action = args.post_action     # the label for testing the deconfouded results\n",
    "    save_name = None\n",
    "    log_file = None\n",
    "    FEA_FEED_LIST = ['item_id','duration_time'] #['item_id','bgm_song_id','bgm_singer_id',\"videoplayseconds\"]\n",
    "    print(\"FEA_FEED_LIST:\", FEA_FEED_LIST, file=log_file)\n",
    "    print(\"**** post_action:\", post_action, file=log_file)\n",
    "    # length_name = 'videoplayseconds'\n",
    "    length_name =  'duration_time' # \"videoplayseconds\"#\n",
    "    code_length_name = 'code_duration' # \"code_videolen\" #\n",
    "    item_name = 'item_id'\n",
    "    if args.use_videolen == 0: # not use any video length information\n",
    "        FEA_FEED_LIST = list(filter(lambda x: x!=length_name, FEA_FEED_LIST))\n",
    "        print(\"please make sure that the video length is not used, used features\",FEA_FEED_LIST,file=log_file)\n",
    "    if args.use_codelen > 0:\n",
    "        FEA_FEED_LIST.append(code_length_name) # use the code lenght in FM \n",
    "    # submit = pd.read_csv(ROOT_PATH + '/coded_test_data.csv')[['userid', 'feedid']]\n",
    "    post_action = args.post_action     # the label for testing the deconfouded results\n",
    "    print(\"**** post_action:\", post_action,file=log_file)\n",
    "    for action in ['finish']:          # ACTION_LIST: for training\n",
    "        train = pd.read_csv(ROOT_PATH + '/'+dataset+f'/final_train_data_for_{action}.csv')\n",
    "        # print(\"origin columns:\",train.columns,file=log_file)\n",
    "        valid = pd.read_csv(ROOT_PATH + '/'+dataset+f'/final_valid_data_for_{action}.csv')\n",
    "        test = pd.read_csv(ROOT_PATH + '/'+dataset+f'/final_test_data_for_{action}.csv')\n",
    "        # train = train.sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "        train.columns = change_columns_name(train.columns.tolist(), uname='user_id', iname='item_id')\n",
    "        valid.columns = change_columns_name(valid.columns.tolist(), uname='user_id', iname='item_id')\n",
    "        test.columns = change_columns_name(test.columns.tolist(), uname='user_id', iname='item_id')\n",
    "        USE_FEAT = ['user_id', 'item_id', action, post_action] + FEA_FEED_LIST[1:]\n",
    "        train= train[USE_FEAT]\n",
    "        test = test[USE_FEAT]\n",
    "        valid = valid[USE_FEAT]\n",
    "        print(\"posi prop:\",sum((train[action]==1)*1)/train.shape[0],file=log_file)\n",
    "        # print()\n",
    "        # test = pd.read_csv(ROOT_PATH + '/test_data.csv')[[i for i in USE_FEAT if i != action and i!=post_action]]\n",
    "        target = [action, post_action] # here we have two task, one for direct target and one post target\n",
    "        data = pd.concat((train, valid, test)).reset_index(drop=True)\n",
    "        if args.use_videolen>0:\n",
    "            dense_features = [length_name]\n",
    "            data[dense_features] = data[dense_features].fillna(0)\n",
    "        else:\n",
    "            dense_features = []\n",
    "        sparse_features = [i for i in USE_FEAT if i not in dense_features and i not in target]\n",
    "        data[sparse_features] = data[sparse_features].fillna(0)\n",
    "        \n",
    "        # 2.count #unique features for each sparse field,and record dense feature field name\n",
    "        if args.use_videolen > 0:\n",
    "            print(\"use video length\",file=log_file)\n",
    "            fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique(),embedding_dim=args.dim)\n",
    "                                    for feat in sparse_features] + [DenseFeat(feat, 1, )\n",
    "                                                                    for feat in dense_features]\n",
    "        else: # not use video lenght information\n",
    "            print(\"not use video length\",file=log_file)\n",
    "            fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique(),embedding_dim=args.dim)\n",
    "                                    for feat in sparse_features]\n",
    "\n",
    "        dnn_feature_columns = fixlen_feature_columns\n",
    "        linear_feature_columns = fixlen_feature_columns\n",
    "\n",
    "        feature_names = get_feature_names(\n",
    "            linear_feature_columns + dnn_feature_columns)\n",
    "\n",
    "        # 3.generate input data for model\n",
    "        train, valid, test = data.iloc[:train.shape[0]].reset_index(drop=True),  data.iloc[train.shape[0]:train.shape[0]+valid.shape[0]].reset_index(drop=True), data.iloc[train.shape[0]+valid.shape[0]:].reset_index(drop=True)\n",
    "        \n",
    "        train_model_input = {name: train[name] for name in feature_names}\n",
    "        valid_model_input = {name: valid[name] for name in feature_names}\n",
    "        test_model_input = {name: test[name] for name in feature_names}\n",
    "        if args.model == 'FairNFM' or args.model =='fastFairNFM' or args.model=='CR_NFM' or args.model=='FairGo' or args.model=='ipw' or args.model==\"DCR_NFM\" or args.model==\"ADCR_NFM\":\n",
    "            confounder = code_length_name\n",
    "            assert confounder == feature_names[-1] # the confounder must be to place at the last column.\n",
    "        print(\"input features:\",train_model_input.keys(),file=log_file)\n",
    "\n",
    "        # 4.Define Model,train,predict and evaluate\n",
    "        device = 'cpu'\n",
    "        use_cuda = True\n",
    "        if use_cuda and torch.cuda.is_available():\n",
    "            # print('cuda ready...',file=log_file)\n",
    "            device = 'cuda:'+ str(args.cuda)\n",
    "       \n",
    "        if args.model == 'deepFM':\n",
    "            used_model = MyDeepFM\n",
    "        elif args.model == 'deepNFM':\n",
    "            used_model = DeepNFM\n",
    "        elif args.model == 'NFM':\n",
    "            used_model = NFM\n",
    "        elif args.model == 'myNFM':\n",
    "            used_model = myNFM\n",
    "        elif args.model =='2FM':\n",
    "            used_model = FM_witout_FirstOrder\n",
    "        elif args.model =='FM':\n",
    "            used_model = MyFM\n",
    "        elif args.model =='CR_NFM':\n",
    "            used_model = CR_NFM\n",
    "        elif args.model == 'ipw':\n",
    "            used_model = ipw_myNFM\n",
    "            vlen_info = data.groupby(confounder).agg({'finish':'mean'})\n",
    "            con = vlen_info.index.values\n",
    "            proensity_pos = vlen_info.values\n",
    "            proensity_neg = 1 - proensity_pos\n",
    "            # proensity_pos = (proensity_pos/proensity_pos.sum())**0.5\n",
    "            # proensity_neg = (proensity_neg/proensity_neg.sum())**0.5\n",
    "            proensity_pos = (proensity_pos/proensity_pos.max())**0.5\n",
    "            proensity_neg = (proensity_neg/proensity_neg.max())**0.5\n",
    "            con_idx = np.argsort(con)\n",
    "            propensity_pos = proensity_pos[con_idx]\n",
    "            propensity_neg = proensity_neg[con_idx]\n",
    "            linear_feature_columns = list(filter(lambda x: x.name != confounder, linear_feature_columns))\n",
    "            dnn_feature_columns = list(filter(lambda x: x.name!= confounder, dnn_feature_columns))\n",
    "            # the confusing feature is not inputted for ipw\n",
    "        elif args.model == 'FairGo':\n",
    "            used_model = FairGo\n",
    "            confounder_num = data[confounder].unique().shape[0]\n",
    "            if args.use_codelen==0:\n",
    "                linear_feature_columns = list(filter(lambda x: x.name != confounder, linear_feature_columns))\n",
    "                dnn_feature_columns = list(filter(lambda x: x.name!= confounder, dnn_feature_columns))\n",
    "        elif args.model == \"DCR_NFM\" or args.model==\"ADCR_NFM\":\n",
    "            if args.model == \"DCR_NFM\":\n",
    "                used_model = DCR_NFM\n",
    "            else:\n",
    "                used_model = DCR_NFM_apr\n",
    "            confounder_num = data[confounder].unique().shape[0]\n",
    "            confounder_pd = train[[item_name,confounder]] #.drop_duplicates('feedid')\n",
    "            # confounder_total = train[confounder].values.reshape(-1)\n",
    "            _, confoudner_prob = np.unique(confounder_pd[confounder].values.reshape(-1),return_counts=True) # compute by the interaction\n",
    "            confoudner_prob = confoudner_prob * 1.0 / confoudner_prob.sum()\n",
    "        elif args.model == 'FairNFM' or args.model =='fastFairNFM':\n",
    "            if args.model == 'FairNFM':\n",
    "                used_model = FairNFM\n",
    "            else:\n",
    "                used_model =FastFairNFM\n",
    "            confounder_num = data[confounder].unique().shape[0]\n",
    "            confounder_pd = train[[item_name,confounder]] #.drop_duplicates('feedid')\n",
    "            # confounder_total = train[confounder].values.reshape(-1)\n",
    "            _, confoudner_prob = np.unique(confounder_pd[confounder].values.reshape(-1),return_counts=True) # compute by the interaction\n",
    "            confoudner_prob = confoudner_prob * 1.0 / confoudner_prob.sum()  # the probability\n",
    "            linear_feature_columns = list(filter(lambda x: x.name != confounder, linear_feature_columns))\n",
    "            dnn_feature_columns = list(filter(lambda x: x.name!= confounder, dnn_feature_columns))   # filter the confouder in the feature that  model will utiliz\n",
    "        else:\n",
    "            print(\"don't have this type model:\", args.model,fiel=log_file)\n",
    "            exit()\n",
    "        \n",
    "        dnn_hidden_units = (256,128)\n",
    "        print(\"used model:\",used_model,file=log_file)\n",
    "        # creat model based on args\n",
    "            \n",
    "        if args.model == 'FairNFM' or args.model =='fastFairNFM':\n",
    "            model = used_model(linear_feature_columns=linear_feature_columns, dnn_hidden_units=dnn_hidden_units, dnn_feature_columns=dnn_feature_columns,\n",
    "                        task='binary', l2_reg_embedding=args.reg_emb, device=device, l2_reg_dnn=args.reg_para, \n",
    "                        l2_reg_linear=args.reg_para,emb_dim=args.dim,confounder_num=confounder_num,confounder_name=confounder,confounder_prob=confoudner_prob)\n",
    "        elif args.model == 'DCR_NFM' or args.model==\"ADCR_NFM\":\n",
    "            model = used_model(linear_feature_columns=linear_feature_columns, dnn_hidden_units=dnn_hidden_units, dnn_feature_columns=dnn_feature_columns,\n",
    "                        task='binary', l2_reg_embedding=args.reg_emb, device=device, l2_reg_dnn=args.reg_para, \n",
    "                        l2_reg_linear=args.reg_para,emb_dim=args.dim,confounder_num=confounder_num,confounder_name=confounder,confounder_prob=confoudner_prob)\n",
    "        elif args.model == 'CR_NFM':\n",
    "            # print(\"load pretrained model...\")\n",
    "            model = used_model(linear_feature_columns=linear_feature_columns,dnn_hidden_units=dnn_hidden_units, dnn_feature_columns=dnn_feature_columns,\n",
    "                        task='binary', l2_reg_embedding=args.reg_emb, device=device, l2_reg_dnn=args.reg_para, \n",
    "                        l2_reg_linear=args.reg_para,emb_dim=args.dim,spurious_feat_name=confounder)\n",
    "        elif args.model == 'FairGo':\n",
    "            # print(\"load pretrained model...\")\n",
    "            model = used_model(linear_feature_columns=linear_feature_columns,dnn_hidden_units=dnn_hidden_units, dnn_feature_columns=dnn_feature_columns,\n",
    "                        task='binary', l2_reg_embedding=args.reg_emb, device=device, l2_reg_dnn=args.reg_para, \n",
    "                        l2_reg_linear=args.reg_para,emb_dim=args.dim,confounder_num=confounder_num,confounder=confounder)\n",
    "            model.load_pretrained_recommender(args.pretrained_model)\n",
    "        elif args.model == 'ipw':\n",
    "            # print(\"load pretrained model...\")\n",
    "            model = used_model(linear_feature_columns=linear_feature_columns,dnn_hidden_units=dnn_hidden_units, dnn_feature_columns=dnn_feature_columns,\n",
    "                        task='binary', l2_reg_embedding=args.reg_emb, device=device, l2_reg_dnn=args.reg_para, \n",
    "                        l2_reg_linear=args.reg_para,emb_dim=args.dim,propensity_pos=propensity_pos,propensity_neg=propensity_neg,confounder_name=confounder)\n",
    "        else:\n",
    "            # print(\"load pretrained model...\")\n",
    "            model = used_model(linear_feature_columns=linear_feature_columns,dnn_hidden_units=dnn_hidden_units, dnn_feature_columns=dnn_feature_columns,\n",
    "                        task='binary', l2_reg_embedding=args.reg_emb, device=device, l2_reg_dnn=args.reg_para, \n",
    "                        l2_reg_linear=args.reg_para,emb_dim=args.dim)\n",
    "\n",
    "        if lr == 0:\n",
    "            lr=None # taking default leaning rate\n",
    "        model.compile(used_optimizer, \"binary_crossentropy\", metrics=[\"binary_crossentropy\"],lr=lr)\n",
    "        \n",
    "        # print(\"the best model will be saved as:\",\"best-\"+save_name, file=log_file)\n",
    "        if need_train:\n",
    "            history = model.fit(train_model_input, train[target].values, batch_size=args.batch_size, epochs=args.epoch, verbose=1,\\\n",
    "                validation_data=(valid_model_input,valid[target].values), save_name=None,args_=args,log_file=log_file)\n",
    "        else:\n",
    "            print(\"test on pretrined model....\",file=log_file)\n",
    "        # given the prediction for test data at the best model\n",
    "        print(\"test model on the best model\")\n",
    "        #save_name = dataset + \"-\" + used_optimizer+\"-\"+args.model+\"-vl-\"+str(args.use_videolen)+'-vc-'+str(args.use_codelen)+'-' + post_action + '-' + args.model+\"-lr_{}-reg_emb_{}-reg_para_{}-dim_{}-stop-{}-auxloss-{}\".format(args.lr,args.reg_emb, args.reg_para, args.dim, args.stop_refer,args.alpha)+\"-stop-\"+args.stop_refer+'-train-'+str(True)\n",
    "        \n",
    "        if os.path.exists(model_path):\n",
    "            try:\n",
    "                model.load_state_dict(torch.load(model_path))\n",
    "            except:\n",
    "                model = torch.load(model_path)\n",
    "        else:\n",
    "            print(\"can not load model...\",)\n",
    "            return 0\n",
    "        \n",
    "        def one_test(test_new):\n",
    "            test_y = test_new[target].values\n",
    "            test_model_input_new = {name: test_new[name] for name in feature_names}\n",
    "            test_metrics  = metrics(test_new['user_id'].values.squeeze(),test_y[:,0])   \n",
    "            test_metrics_post  = metrics(test_new['user_id'].values.squeeze(),test_y[:,1])  \n",
    "            if args.model == 'FairNFM' or args.model =='fastFairNFM': # FairNFM\n",
    "                mode_list = mode #['condition','do','do-even']\n",
    "                for pre_mode in mode_list:\n",
    "                    print(\"\\n||| **********prediction mode:\",pre_mode,file=log_file)\n",
    "                    model.set_pre_mode(mode=pre_mode)\n",
    "                    pred_ans = model.predict(test_model_input_new, 2048)\n",
    "                    # uauc1,_,_ = uAUC(test['user_id'],pred_ans,test_y[:,0])\n",
    "                    topK = [10,20,50]\n",
    "                    # uauc, map_list, ndcg_list = test_metrics.test2(pred_ans,topK=topK)\n",
    "                    # print('test with the best model, finish uauc:', uauc, 'map:', map_list,'ndcg:', ndcg_list, file=log_file)\n",
    "                    # uauc2,_,_ = uAUC(test['user_id'].values, pred_ans, test_y[:,1])\n",
    "                    uauc_post, map_post_list, ndcg_post_list = test_metrics_post.test2(pred_ans,topK=topK)\n",
    "                    print('test with the best model, post action uauc', uauc_post, 'map:', map_post_list, 'ndcg', ndcg_post_list, file=log_file)\n",
    "                    torch.cuda.empty_cache()\n",
    "            else:\n",
    "                pred_ans = model.predict(test_model_input_new, 2048)\n",
    "                test_y = test[target].values\n",
    "                topK = [10,20,50]\n",
    "                # uauc, map_list, ndcg_list = test_metrics.test2(pred_ans,topK=topK)\n",
    "                # print('test with the best model, finish uauc:', uauc, 'map:', map_list,'ndcg:', ndcg_list,file=log_file)\n",
    "                # uauc2,_,_ = uAUC(test['user_id'].values, pred_ans, test_y[:,1])\n",
    "                uauc_post, map_post_list, ndcg_post_list = test_metrics_post.test2(pred_ans,topK=topK)\n",
    "                print('test with the best model, post action uauc', uauc_post, 'map:', map_post_list, 'ndcg', ndcg_post_list, file=log_file)\n",
    "                # uauc1,_,_ = uAUC(test['user_id'],pred_ans,test_y[:,0])\n",
    "                # print('test with the best model, finish uauc:', uauc1)\n",
    "                # uauc2,_,_ = uAUC(test['user_id'].values, pred_ans, test_y[:,1])\n",
    "                # print('test with the best model, like uauc:',uauc2)\n",
    "                torch.cuda.empty_cache()\n",
    "        print(\"\\n------------valid--------\")\n",
    "        one_test(valid)\n",
    "        print(\"\\n------------test------------\")\n",
    "        one_test(test)\n",
    "\n",
    "        user_info = data.groupby('user_id').agg({\"finish\":'count', post_action:'sum'})\n",
    "        s_action = post_action # 'finish' #\n",
    "        active_user1 = user_info[user_info[s_action]>3].index.values\n",
    "        active_user2 = user_info[user_info['finish']>=700].index.values\n",
    "        active_user = np.intersect1d(active_user1, active_user2) \n",
    "        nonactive_user = np.setdiff1d(user_info.index.values, active_user)\n",
    "\n",
    "\n",
    "        print(\"act like number:\",data[data['user_id'].isin(active_user)][post_action].sum(),\"user number:\",active_user.shape)\n",
    "        print(\"non act like number:\",data[data['user_id'].isin(nonactive_user)][post_action].sum(),\"non active num:\",nonactive_user.shape)\n",
    "        print(\"\\n------------actviate test..\")\n",
    "        test_active = test[test['user_id'].isin(active_user)]\n",
    "        one_test(test_active)\n",
    "        print(\"\\n------------non activate test...\")\n",
    "        test_nonactivate = test[test['user_id'].isin(nonactive_user)]\n",
    "        one_test(test_nonactivate)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ipw min-max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'ipw_nfm.ipw_myNFM'>\n",
      "propensity need grad?:  False\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "test user num: 9180\n",
      "test with the best model, post action uauc [0.08028007 0.15199883 0.37001101] map: [0.026824   0.0327344  0.04358902] ndcg [0.04829131 0.0707976  0.12504686]\n",
      "\n",
      "------------test------------\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.07748418 0.14739648 0.36645882] map: [0.02496854 0.03078361 0.0417176 ] ndcg [0.04643403 0.06835173 0.12294264]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.03489684 0.06508574 0.16664261] map: [0.010757   0.01268173 0.01689567] ndcg [0.02398548 0.03443508 0.06305565]\n",
      "\n",
      "------------non activate test...\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.10289898 0.19651695 0.48570285] map: [0.03344954 0.04158624 0.05653055] ndcg [0.05983062 0.08859213 0.15868132]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='ipw'\n",
    "model_path = \"/data/zyang/decFair/logs/best-kwai-max_p_adagrad-ipw-vl-0-vc-1-like-ipw-lr_0.01-reg_emb_0-reg_para_0.001-dim_16-stop-val_ndcg_post10-auxloss-0-stop-val_ndcg_post10-train-True-m.pth\"\n",
    "run_a_model(config,model_path=model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DecRec: DCR-MOE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'decfair_tune.FastFairNFM'>\n",
      "read embedding dim: 17\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "\n",
      "------------test------------\n",
      "\n",
      "||| **********prediction mode: do\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.10891465 0.19361961 0.40954135] map: [0.03535654 0.04233912 0.05339821] ndcg [0.0633834  0.08958259 0.14395099]\n",
      "\n",
      "||| **********prediction mode: do-even\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.10867256 0.19484525 0.40896165] map: [0.0352112  0.04232559 0.05326489] ndcg [0.06326869 0.08984544 0.14372838]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "\n",
      "||| **********prediction mode: do\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.05705968 0.098574   0.22653742] map: [0.01859742 0.02131539 0.02691425] ndcg [0.03760664 0.05153201 0.08716922]\n",
      "\n",
      "||| **********prediction mode: do-even\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.05717982 0.09812579 0.22558492] map: [0.01838722 0.02107224 0.02659259] ndcg [0.03752241 0.05117789 0.08661333]\n",
      "\n",
      "------------non activate test...\n",
      "\n",
      "||| **********prediction mode: do\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.1397732  0.25044987 0.51857861] map: [0.04535998 0.05490259 0.06921286] ndcg [0.07875268 0.11232021 0.17781535]\n",
      "\n",
      "||| **********prediction mode: do-even\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.13940181 0.25256439 0.51839513] map: [0.04525121 0.05500883 0.069182  ] ndcg [0.07863326 0.11292079 0.17781264]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='fastFairNFM'\n",
    "model_path = \"/data/zyang/decFair/best_model/best-kwai-rerunadagrad-fastFairNFM-vl-0-vc-1-like-fastFairNFM-lr_0.001-reg_emb_1e-06-reg_para_0.01-dim_16-stop-val_do_ndcg_post10-auxloss-0-stop-val_do_ndcg_post10-train-True-m2.pth\"\n",
    "run_a_model(config,model_path=model_path,mode=['do','do-even'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Condition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'decfair_tune.FastFairNFM'>\n",
      "read embedding dim: 17\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "\n",
      "------------test------------\n",
      "\n",
      "||| **********prediction mode: condition\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.07573916 0.14782236 0.37395606] map: [0.02399072 0.03008448 0.04153916] ndcg [0.044766   0.06772146 0.12431245]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "\n",
      "||| **********prediction mode: condition\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.03860239 0.07424811 0.17758887] map: [0.01200973 0.014193   0.01869143] ndcg [0.02591287 0.0379584  0.06751442]\n",
      "\n",
      "------------non activate test...\n",
      "\n",
      "||| **********prediction mode: condition\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.09764662 0.19187393 0.49114184] map: [0.03097164 0.03943253 0.05503229] ndcg [0.05585452 0.08544637 0.15812852]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='fastFairNFM'\n",
    "model_path = \"/data/zyang/decFair/best_model/best-kwai-adagrad-fastFairNFM-vl-0-vc-1-like-fastFairNFM-lr_0.01-reg_emb_0.0001-reg_para_0.0001-dim_16-stop-val_condition_ndcg_post10-auxloss-0-stop-val_condition_ndcg_post10-train-True-m2.pth\"\n",
    "run_a_model(config, model_path=model_path,mode=['condition'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NFM-WOA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'baseline_tune.NFM'>\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "\n",
      "------------test------------\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.07593069 0.14942205 0.37001938] map: [0.02282427 0.02897112 0.04012936] ndcg [0.04394405 0.06719739 0.12259465]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.03744576 0.07293121 0.17386852] map: [0.01116191 0.01352693 0.01800183] ndcg [0.02493237 0.03698578 0.06617582]\n",
      "\n",
      "------------non activate test...\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.09889729 0.19506938 0.48707605] map: [0.029784   0.03818773 0.05333437] ndcg [0.05528962 0.08522673 0.15626364]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='NFM'\n",
    "model_path = \"/data/zyang/decFair/best_model/best-kwai-adagrad-NFM-vl-0-vc-0-like-NFM-lr_0.01-reg_emb_0-reg_para_1e-06-dim_16-stop-val_ndcg_post10-auxloss-1.0-stop-val_ndcg_post10-m.pt\"\n",
    "run_a_model(config, model_path=model_path,mode=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# myNFM: NFM-WA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'FairGo.myNFM'>\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "\n",
      "------------test------------\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.0778257  0.15301037 0.37727873] map: [0.02468396 0.03098074 0.04226476] ndcg [0.04579407 0.06938347 0.12537908]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.03968335 0.07387359 0.17605479] map: [0.01177384 0.01393398 0.01840033] ndcg [0.02586756 0.03741014 0.0666527 ]\n",
      "\n",
      "------------non activate test...\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.10045901 0.20029463 0.49736285] map: [0.03243311 0.04121773 0.05657068] ndcg [0.05767384 0.08850824 0.16045513]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='myNFM'\n",
    "model_path = \"/data/zyang/decFair/logs/best-kwai-adagrad-myNFM-vl-0-vc-1-like-myNFM-lr_0.01-reg_emb_0.001-reg_para_0-dim_16-stop-val_ndcg_post10-auxloss-1.0-stop-val_ndcg_post10-m.pth\" \n",
    "run_a_model(config, model_path=model_path,mode=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CR_NFM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'counterfactual_reasoning.CR_NFM'>\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "\n",
      "------------test------------\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.07643817 0.14915611 0.37267365] map: [0.02435698 0.03051063 0.04173107] ndcg [0.04577748 0.06866736 0.124431  ]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.03594741 0.06726998 0.17709978] map: [0.01132873 0.01344432 0.01815836] ndcg [0.02489044 0.03586634 0.06661868]\n",
      "\n",
      "------------non activate test...\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.10060179 0.19802318 0.489386  ] map: [0.03213183 0.04069527 0.05579847] ndcg [0.0582422  0.08824198 0.15893153]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='CR_NFM'\n",
    "model_path = \"/data/zyang/decFair/best_model/best-kwai-adagrad-CR_NFM-vl-0-vc-1-like-CR_NFM-lr_0.01-reg_emb_0-reg_para_0.0001-dim_16-stop-val_ndcg_post10-auxloss-0.5-stop-val_ndcg_post10-m.pth\"\n",
    "run_a_model(config, model_path=model_path,mode=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FairGo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: kwai\n",
      "FEA_FEED_LIST: ['item_id', 'duration_time']\n",
      "**** post_action: like\n",
      "please make sure that the video length is not used, used features ['item_id']\n",
      "**** post_action: like\n",
      "posi prop: 0.11959629148971158\n",
      "not use video length\n",
      "input features: dict_keys(['user_id', 'item_id', 'code_duration'])\n",
      "used model: <class 'FairGo.FairGo'>\n",
      "test on pretrined model....\n",
      "test model on the best model\n",
      "\n",
      "------------valid--------\n",
      "\n",
      "------------test------------\n",
      "test user num: 9191\n",
      "test with the best model, post action uauc [0.08759747 0.16514625 0.39125038] map: [0.02687649 0.03321764 0.04450417] ndcg [0.05061749 0.0749184  0.13141825]\n",
      "act like number: 72195 user number: (4176,)\n",
      "non act like number: 64490 non active num: (13843,)\n",
      "\n",
      "------------actviate test..\n",
      "test user num: 3435\n",
      "test with the best model, post action uauc [0.03718318 0.07682204 0.18986323] map: [0.01157662 0.01432612 0.01915917] ndcg [0.02528577 0.03890393 0.07084123]\n",
      "\n",
      "------------non activate test...\n",
      "test user num: 5756\n",
      "test with the best model, post action uauc [0.11347007 0.21819385 0.5118717 ] map: [0.03462924 0.0435707  0.0587496 ] ndcg [0.06308454 0.09499619 0.16633647]\n"
     ]
    }
   ],
   "source": [
    "config={}\n",
    "config['model']='FairGo'\n",
    "model_path = \"/data/zyang/decFair/best_model/best-kwai-adagrad-FairGo-vl-0-vc-1-like-FairGo-lr_0.01-reg_emb_0-reg_para_0.001-dim_16-stop-val_ndcg_post10-auxloss-0.01-stop-val_ndcg_post10-train-True-m.pth\"\n",
    "run_a_model(config, model_path=model_path,mode=None)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4775800764512356c297cc90964435a04e49084d3ccf324f38a0ce1693d6b1fd"
  },
  "kernelspec": {
   "display_name": "Python 3.8.0 64-bit ('torch': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}