feature_gen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Breast Cancer Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Credit to Tyler Spears and Sonia Baee, who started the original pipeline code (eg. reading in files)\n",
    "\n",
    "# imports\n",
    "import sys\n",
    "import os\n",
    "import functools\n",
    "import pathlib\n",
    "import glob\n",
    "import collections\n",
    "import itertools\n",
    "import re\n",
    "import random\n",
    "try:\n",
    "    import cPickle as pickle\n",
    "except ModuleNotFoundError:\n",
    "    import pickle\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pipeline\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "\n",
    "import scipy\n",
    "from scipy.spatial.distance import cdist\n",
    "\n",
    "# visualization libraries\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "from matplotlib.dates import DateFormatter\n",
    "import matplotlib.dates as mdates\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams.update({'figure.autolayout': True})\n",
    "plt.rcParams.update({'figure.facecolor': [1.0, 1.0, 1.0, 1.0]})\n",
    "\n",
    "# configure autoreloading of modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_dataset = pipeline.import_processed_files()\n",
    "master_featureset = pipeline.create_file_dictionary('features')['features']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demographic & Mood Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize featureset to be the Pre- and Post- measures\n",
    "features = master_dataset['pre_post']\n",
    "features.drop('dataset', axis=1, inplace=True)\n",
    "\n",
    "# Add helper columns denoting who has app data and who dropped out\n",
    "app_users = list(master_dataset['app_launch']['pid'].unique())\n",
    "features['has_app_data'] = features['pid'].apply(lambda x: x in app_users)\n",
    "features['dropped'] = [pipeline.match_value(master_dataset['dropouts'], 'pid', pid, 'dropped') for pid in features['pid']]\n",
    "features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Baseline PHQ-4 & PROMISE Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Denote that the baseline was the PHQ4, specifically\n",
    "features.rename(index=str, columns={'phq_bl': 'phq4_bl', 'phq_post': 'phq4_post'}, inplace=True)\n",
    "# features.drop(columns=['phq4'], inplace=True)\n",
    "\n",
    "# Create new columns for phq4 subscales\n",
    "df = master_dataset['blsurvey']\n",
    "\n",
    "for subscale, descriptors in pipeline.PHQ4_SCORING['subscales'].items():\n",
    "    cols = descriptors['cols']\n",
    "    for col in cols:\n",
    "        df[col] = df[col].map(descriptors['codes'])\n",
    "    \n",
    "    new_col = 'phq4_' + str(subscale) + '_bl'\n",
    "    df[new_col] = df.apply(lambda row: row[cols].sum(),axis=1)\n",
    "    temp = df.loc[:, ['pid', new_col]]\n",
    "    features = features.set_index('pid').join(temp.set_index('pid')).reset_index()\n",
    "\n",
    "    \n",
    "# Divide users into baseline 'low' or 'high' depressed and 'low' or 'high' anxious groups,\n",
    "# based on PHQ4 scoring and PROMIS t-score cut points\n",
    "\n",
    "features['trait_dep_group'] = np.where((features['phq4_depression_bl'] >= pipeline.PHQ4_THRESH) | (features['promis_dep_bl'] >= pipeline.PROMIS_THRESH), 'high', 'low')\n",
    "features['trait_anx_group'] = np.where((features['phq4_anxiety_bl'] >= pipeline.PHQ4_THRESH) | (features['promis_anx_bl'] >= pipeline.PROMIS_THRESH), 'high', 'low')\n",
    "\n",
    "features.head(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weekly Mood Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Associate weekly survey responses with weeks of the study\n",
    "# df = master_dataset['wklysurvey']\n",
    "# df['enddate'] = pd.to_datetime(df['enddate'])\n",
    "\n",
    "timestamps = master_dataset['wklysurvey_timestamps']\n",
    "timestamps['date'] = pd.to_datetime(timestamps['date'])\n",
    "\n",
    "# df['weekofstudy'] = [pipeline.find_week_by_timestamp(master_dataset['wklysurvey_timestamps'], \n",
    "#                                                      row['pid'], row['enddate']) \n",
    "#                      for index, row in df.iterrows()]\n",
    "# pd.set_option('display.max_rows', 180)\n",
    "\n",
    "# df.to_csv('data/processed/wklysurvey_processed.csv')\n",
    "# df[['pid', 'startdate', 'enddate','weekofstudy']]\n",
    "\n",
    "# IMPORTANT: If NaNs in weekofstudy col, fix manually and re-import dataset!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Run only after manual correction, if needed\n",
    "# master_dataset['wklysurvey'] = pd.read_csv('data/processed/wklysurvey_processed.csv')\n",
    "master_dataset['wklysurvey'][['pid', 'startdate', 'enddate','weekofstudy']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store response rates for the weekly surveys, for later analysis\n",
    "df = timestamps.groupby('method')['method'].count().reset_index(name='num_users')\n",
    "# reminder_types\n",
    "\n",
    "# df = timestamps.groupby('method')\n",
    "response_rates = timestamps.groupby('method')['completed'].sum() / timestamps.groupby('method')['sent'].sum()\n",
    "response_rates = response_rates.reset_index(name=\"response_rate\")\n",
    "\n",
    "df = df.merge(response_rates, on=\"method\")\n",
    "df.to_csv('features/response_rates.csv')\n",
    "df\n",
    "# res.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct weekly mood scores (anxiety and depression)\n",
    "\n",
    "mood = {}\n",
    "for c in ['anx', 'dep']:\n",
    "    \n",
    "    # Extract weekly mood score\n",
    "    df = master_dataset['wklysurvey']\n",
    "\n",
    "    for i in range(2, 8):\n",
    "        mood_df = df.loc[df['weekofstudy'] == i][['pid', c]]\n",
    "        mood_df.columns = ['pid', 'w' + str(i) + '_' + c]\n",
    "        mood[i] = mood_df\n",
    "\n",
    "    for week, mood_df in mood.items():\n",
    "        features = features.set_index('pid').join(mood_df.set_index('pid')).reset_index()\n",
    "\n",
    "    # Add the week1 score (mean of 7 total daily mood scores)\n",
    "    df = master_dataset['fwsurveys']\n",
    "    df = df.groupby('pid')[c].mean().round(0).reset_index(name='w1' +  '_' + c)\n",
    "    features = features.merge(df, on=\"pid\")   \n",
    "    \n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to CSV\n",
    "features.to_csv('features/' + pipeline.ALL_USERS_DIR + 'pre_post.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### App Usage Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we compute features related to app usage, we should tidy up the app_launch data table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_launch = master_dataset['app_launch']\n",
    "\n",
    "# Eliminate launches with duration < 5 seconds\n",
    "app_launch = app_launch[app_launch['duration'] > 0]\n",
    "\n",
    "app_launch['date'] = pd.to_datetime(app_launch['date'], errors='coerce')\n",
    "app_launch['day'] = app_launch['date'].dt.day\n",
    "app_launch['week'] = app_launch['date'].dt.week\n",
    "app_launch['hour'] = app_launch['date'].dt.hour\n",
    "\n",
    "# Identify the participant's start date\n",
    "app_launch['startdate'] = [pipeline.match_value(master_dataset['blsurvey'], 'pid', x, 'startdate') for x in app_launch['pid']]\n",
    "\n",
    "# Identify epoch (6 hour time window) during which each app launch occured\n",
    "app_launch['epoch'] = pd.cut(app_launch['hour'], pipeline.EPOCHS['bins'], labels=pipeline.EPOCHS['labels'])\n",
    "\n",
    "# Determine the day and week of study, for each observation, \n",
    "# based on the participant's start date\n",
    "app_launch['timeelapsed'] = pd.to_datetime(app_launch['date']) - pd.to_datetime(app_launch['startdate'])\n",
    "app_launch['weekofstudy'] = np.ceil(app_launch['timeelapsed'].dt.days / 7.0)\n",
    "app_launch['weekofstudy'] = app_launch['weekofstudy'].astype(int)\n",
    "app_launch['dayofstudy'] = np.ceil(app_launch['timeelapsed'].dt.days)\n",
    "app_launch['dayofweek'] = app_launch['date'].dt.dayofweek\n",
    "\n",
    "# Keep only the data from the weeks of the study\n",
    "app_launch = app_launch[(1 <= app_launch['weekofstudy']) & (app_launch['weekofstudy'] <= 7)]\n",
    "\n",
    "# Extract list of apps\n",
    "apps = list(app_launch['package'].unique())\n",
    "app_launch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save out to csv, to save time later\n",
    "app_launch.to_csv('data/processed/app_launch_processed.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weekly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wkly_moods = master_dataset['wklysurvey'][['pid', 'weekofstudy', 'anx', 'dep']]\n",
    "\n",
    "def add_affect(df, features, timediv, wkly_affect_df=None) :\n",
    "        \n",
    "    if timediv == 'wkly':\n",
    "        # Record wkly affect by week\n",
    "        df = pd.merge(df, wkly_affect_df, on=['pid', 'weekofstudy'], how=\"left\")\n",
    "    \n",
    "    # Record trait affect groups for anxiety and depression\n",
    "    for group in pipeline.TRAIT_AFFECT_GROUPS.keys():\n",
    "        df[group] = df['pid'].apply(lambda x:  pipeline.match_value(features, 'pid', x, group))\n",
    "        \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applevel\n",
    "ind_applevel = app_launch.groupby(['pid','package'])['weekofstudy'].value_counts().reset_index(name='frequency')\n",
    "df = app_launch.groupby(['pid','package', 'weekofstudy'])['dayofweek'].nunique().reset_index(name='daysofuse')\n",
    "\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = list(df.columns)[:-1],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_noepoch(app_launch, groupbycols = ['pid','package','weekofstudy'])\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = list(df.columns)[:-7],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "ind_agg = app_launch.groupby('pid')['weekofstudy'].value_counts().reset_index(name='frequency')\n",
    "\n",
    "df = app_launch.groupby(['pid','weekofstudy'])['dayofweek'].nunique().reset_index(name='daysofuse')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = list(df.columns)[:-1],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_noepoch(app_launch, groupbycols = ['pid','weekofstudy'])\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = list(df.columns)[:-7],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'wkly', wkly_moods)\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'wkly', wkly_moods)\n",
    "\n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'wkly_applevel.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'wkly_agg.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weekly x Time of Day\n",
    "**IMPORTANT NOTE:** Throughout the code, I've referred to the time of day as the \"epoch\". However, note that a week is also considered an epoch, by our own definition in the paper. Probably need to change this terminology, at some point. - Anna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_cols=['pid', 'package', 'weekofstudy']\n",
    "\n",
    "# Applevel\n",
    "df = app_launch.groupby(['pid', 'package', 'weekofstudy'])['epoch'].value_counts()\n",
    "ind_applevel = pipeline.weekly_epoch_breakdown(df, 'frequency', merge_cols)\n",
    "\n",
    "df = app_launch.groupby(['pid','package','weekofstudy','epoch'])['dayofweek'].nunique()\n",
    "df = pipeline.weekly_epoch_breakdown(df, 'daysofuse', merge_cols)\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = merge_cols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_has_epoch(app_launch, groupbycols = ['pid','package','weekofstudy','epoch'])\n",
    "df\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = merge_cols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "merge_cols=['pid', 'weekofstudy']\n",
    "\n",
    "df = app_launch.groupby(['pid', 'weekofstudy'])['epoch'].value_counts()\n",
    "ind_agg = pipeline.weekly_epoch_breakdown(df, 'frequency', merge_cols)\n",
    "\n",
    "df = app_launch.groupby(['pid','weekofstudy','epoch'])['dayofweek'].nunique()\n",
    "df = pipeline.weekly_epoch_breakdown(df, 'daysofuse', merge_cols)\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = ['pid','weekofstudy'],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_has_epoch(app_launch, groupbycols = ['pid','weekofstudy','epoch'])\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = merge_cols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'wkly', wkly_moods)\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'wkly', wkly_moods)\n",
    "\n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'wkly_epoch_applevel.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'wkly_epoch_agg.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weekly x Time of Day (Long Form?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groupbycols=['pid', 'package', 'weekofstudy', 'epoch']\n",
    "\n",
    "# Applevel\n",
    "ind_applevel = app_launch.groupby(\n",
    "    groupbycols[:-1]\n",
    ")['epoch'].value_counts().reset_index(name=\"frequency\")\n",
    "\n",
    "ind_applevel\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['dayofweek'].nunique().reset_index(name='daysofuse')\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['duration'].sum().reset_index(name='duration')\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "groupbycols=['pid', 'weekofstudy', 'epoch']\n",
    "\n",
    "\n",
    "# Applevel\n",
    "ind_agg = app_launch.groupby(\n",
    "    groupbycols[:-1]\n",
    ")['epoch'].value_counts().reset_index(name=\"frequency\")\n",
    "\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['dayofweek'].nunique().reset_index(name='daysofuse')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['duration'].sum().reset_index(name='duration')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'wkly', wkly_moods)\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'wkly', wkly_moods)\n",
    "                \n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'wkly_epoch_applevel_lf.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'wkly_epoch_agg_lf.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Entire Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applevel\n",
    "ind_applevel = app_launch.groupby('pid')['package'].value_counts().reset_index(name='frequency')\n",
    "df = app_launch.groupby(['pid','package'])['dayofstudy'].nunique().reset_index(name='daysofuse')\n",
    "\n",
    "df = pipeline.calc_duration_noepoch(app_launch, groupbycols = ['pid','package'])\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = list(df.columns)[:-7],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "ind_agg = app_launch['pid'].value_counts().reset_index(name=\"frequency\")\n",
    "ind_agg.rename(columns={'index': 'pid'}, inplace=True)\n",
    "\n",
    "df = app_launch.groupby('pid')['dayofstudy'].nunique().reset_index(name='daysofuse')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = list(df.columns)[:-1],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_noepoch(app_launch, groupbycols = ['pid'])\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = list(df.columns)[:-7],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = ind_applevel.round(0)\n",
    "ind_agg = ind_agg.round(0)\n",
    "\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'study')\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'study')\n",
    "                \n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_applevel.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_agg.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Entire Study (Long form?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groupbycols=['pid', 'package']\n",
    "\n",
    "# Applevel\n",
    "ind_applevel = app_launch.groupby(\n",
    "    groupbycols[:-1]\n",
    ")[groupbycols[-1]].value_counts().reset_index(name=\"frequency\")\n",
    "\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['dayofstudy'].nunique().reset_index(name='daysofuse')\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['duration'].sum().reset_index(name='duration')\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "groupbycols=['pid']\n",
    "\n",
    "ind_agg = app_launch['pid'].value_counts().reset_index(name='frequency')\n",
    "ind_agg.rename(columns={'index':'pid'}, inplace=True)\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['dayofstudy'].nunique().reset_index(name='daysofuse')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['duration'].sum().reset_index(name='duration')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'study')\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'study')\n",
    "                \n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_applevel_lf.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_agg_lf.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Study x Time of Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_cols=['pid', 'package']\n",
    "\n",
    "# Applevel\n",
    "df = app_launch.groupby(['pid', 'package'])['epoch'].value_counts()\n",
    "ind_applevel = pipeline.weekly_epoch_breakdown(df, 'frequency', merge_cols)\n",
    "\n",
    "df = app_launch.groupby(['pid','package','epoch'])['dayofstudy'].nunique()\n",
    "df = pipeline.weekly_epoch_breakdown(df, 'daysofuse', merge_cols)\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = merge_cols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_has_epoch(app_launch, groupbycols = ['pid','package','epoch'])\n",
    "df\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = merge_cols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "merge_cols=['pid']\n",
    "\n",
    "df = app_launch.groupby('pid')['epoch'].value_counts()\n",
    "ind_agg = pipeline.weekly_epoch_breakdown(df, 'frequency', merge_cols)\n",
    "\n",
    "df = app_launch.groupby(['pid','epoch'])['dayofstudy'].nunique()\n",
    "df = pipeline.weekly_epoch_breakdown(df, 'daysofuse', merge_cols)\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = ['pid'],\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = pipeline.calc_duration_has_epoch(app_launch, groupbycols = ['pid','epoch'])\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = merge_cols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = ind_applevel.round(0)\n",
    "ind_agg = ind_agg.round(0)\n",
    "\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'study')\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'study')\n",
    "                \n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_epoch_applevel.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_epoch_agg.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind_agg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Study x Time of Day (Long Form?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groupbycols=['pid', 'package', 'epoch']\n",
    "\n",
    "# Applevel\n",
    "ind_applevel = app_launch.groupby(groupbycols[:-1])[groupbycols[-1]].value_counts().reset_index(name=\"frequency\")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['dayofstudy'].nunique().reset_index(name='daysofuse')\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['duration'].sum().reset_index(name='duration')\n",
    "ind_applevel = pd.merge(\n",
    "    ind_applevel, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "# Aggregate\n",
    "groupbycols=['pid', 'epoch']\n",
    "\n",
    "\n",
    "# Applevel\n",
    "ind_agg = app_launch.groupby(groupbycols[:-1])[groupbycols[-1]].value_counts().reset_index(name=\"frequency\")\n",
    "\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['dayofstudy'].nunique().reset_index(name='daysofuse')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "df = app_launch.groupby(groupbycols)['duration'].sum().reset_index(name='duration')\n",
    "ind_agg = pd.merge(\n",
    "    ind_agg, \n",
    "    df, \n",
    "    on = groupbycols,\n",
    "    how=\"outer\"\n",
    ")\n",
    "\n",
    "ind_applevel = add_affect(ind_applevel.round(0), features, 'study')\n",
    "ind_agg =  add_affect(ind_agg.round(0), features, 'study')\n",
    "                \n",
    "ind_applevel.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_epoch_applevel_lf.csv')\n",
    "ind_agg.to_csv('features/' + pipeline.APP_USERS_DIR + 'study_epoch_agg_lf.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind_applevel.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_dataset.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct Weekly Feature Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wkly_vectors = pipeline.construct_feature_vectors(app_launch, master_dataset['wklysurvey'], 'wkly')\n",
    "wkly_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wkly_vectors.to_csv('features/' + pipeline.APP_USERS_DIR + '/all_ind_wkly.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Entire Study Stats (Added 9/8 as Additional Feature Gen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = pd.DataFrame()\n",
    "features = pd.read_csv('features/' + pipeline.APP_USERS_DIR + '/study_agg.csv')\n",
    "cols = ['frequency', 'daysofuse', 'duration']\n",
    "\n",
    "for trait_group, trait_group_label in pipeline.TRAIT_AFFECT_GROUPS.items():\n",
    "    for col in cols:    \n",
    "        to_merge = []\n",
    "        \n",
    "        if col == 'duration':\n",
    "            features[col] = features[col] / 60.0\n",
    "        \n",
    "        df = features.groupby(trait_group)[col].mean().reset_index(name='mean')\n",
    "        \n",
    "        df2 = features.groupby(trait_group)[col].std().reset_index(name='std')\n",
    "        to_merge.append(df2)\n",
    "\n",
    "        df2 = features.groupby(trait_group)[col].var().reset_index(name='var')\n",
    "        to_merge.append(df2)\n",
    "        \n",
    "        for df_to_merge in to_merge:\n",
    "            df = pd.merge(df, df_to_merge, on=trait_group, how=\"outer\")\n",
    "            \n",
    "        df.rename(columns={trait_group: 'trait_group'}, inplace=True)\n",
    "        df['trait_group'] = df['trait_group'].apply(lambda x: x.capitalize())\n",
    "        \n",
    "        df.insert(0, 'trait_affect', np.nan)\n",
    "        if 'anx' in trait_group:\n",
    "            df['trait_affect'] = 'Anxiety'\n",
    "        else:\n",
    "            df['trait_affect'] = 'Depression'\n",
    "\n",
    "        df.insert(2, 'measure', np.nan)\n",
    "        df['measure'] = col\n",
    "        df['measure'] = df['measure'].apply(\n",
    "            lambda x: pipeline.WKLY_AFFECT[x] if x in pipeline.WKLY_AFFECT.keys() else x\n",
    "        )\n",
    "        \n",
    "        if stats.empty:\n",
    "            stats = df\n",
    "        else:\n",
    "            stats = stats.append(df, ignore_index=True)\n",
    "stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.to_csv('features/' + pipeline.APP_USERS_DIR + '/trait_study_stats.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trait Affect x Wkly Stats (Added 9/8 as Additional Feature Gen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = pd.DataFrame()\n",
    "features = pd.read_csv('features/wkly_agg.csv')\n",
    "cols = list(pipeline.WKLY_AFFECT.keys()) + ['frequency', 'daysofuse', 'duration']\n",
    "\n",
    "for trait_group, trait_group_label in pipeline.TRAIT_AFFECT_GROUPS.items():\n",
    "    for col in cols:    \n",
    "        to_merge = []\n",
    "        \n",
    "        if col == 'duration':\n",
    "            features[col] = features[col] / 60.0\n",
    "        \n",
    "        df = features.groupby(trait_group)[col].mean().reset_index(name='mean')\n",
    "        \n",
    "        df2 = features.groupby(trait_group)[col].std().reset_index(name='std')\n",
    "        to_merge.append(df2)\n",
    "\n",
    "        df2 = features.groupby(trait_group)[col].var().reset_index(name='var')\n",
    "        to_merge.append(df2)\n",
    "        \n",
    "        for df_to_merge in to_merge:\n",
    "            df = pd.merge(df, df_to_merge, on=trait_group, how=\"outer\")\n",
    "            \n",
    "        df.rename(columns={trait_group: 'trait_group'}, inplace=True)\n",
    "        df['trait_group'] = df['trait_group'].apply(lambda x: x.capitalize())\n",
    "        \n",
    "        df.insert(0, 'trait_affect', np.nan)\n",
    "        if 'anx' in trait_group:\n",
    "            df['trait_affect'] = 'Anxiety'\n",
    "        else:\n",
    "            df['trait_affect'] = 'Depression'\n",
    "\n",
    "        df.insert(2, 'measure', np.nan)\n",
    "        df['measure'] = col\n",
    "        df['measure'] = df['measure'].apply(\n",
    "            lambda x: pipeline.WKLY_AFFECT[x] if x in pipeline.WKLY_AFFECT.keys() else x\n",
    "        )\n",
    "        \n",
    "        if stats.empty:\n",
    "            stats = df\n",
    "        else:\n",
    "            stats = stats.append(df, ignore_index=True)\n",
    "stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.to_csv('features' + pipeline.APP_USERS_DIR + '/trait_wkly_stats.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = master_dataset['wklysurvey'][['pid','weekofstudy','anx', 'dep']]\n",
    "df = add_affect(df, features, 'study')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('features/all_users/wkly_trait_state.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:int-eng] *",
   "language": "python",
   "name": "conda-env-int-eng-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}