preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/.DS_Store\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Economics.pdf\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/ReadMe.pdf\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/CropParameters.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/Resources.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/Production.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/GreenhouseClimate.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/TomQuality.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/GrodanSens.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Digilog/LabAnalysis.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/CropParameters.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/Resources.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/Production.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/GreenhouseClimate.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/TomQuality.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/GrodanSens.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/IUACAAS/LabAnalysis.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/CropParameters.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/Resources.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/Production.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/GreenhouseClimate.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/TomQuality.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/GrodanSens.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Automatoes/LabAnalysis.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/CropParameters.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/Resources.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/Production.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/GreenhouseClimate.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/TomQuality.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/GrodanSens.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/TheAutomators/LabAnalysis.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Weather/Weather.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/CropParameters.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/Resources.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/Production.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/GreenhouseClimate.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/TomQuality.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/GrodanSens.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/AICU/LabAnalysis.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/CropParameters.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/Resources.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/Production.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/GreenhouseClimate.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/TomQuality.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/GrodanSens.csv\n",
      "/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Reference/LabAnalysis.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from typing import Tuple,Dict\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "\n",
    "directory_path = '/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/'\n",
    "\n",
    "for dirname, _, filenames in os.walk(directory_path):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge Data\n",
    "\n",
    "The input data currently contains multiple csv files \n",
    "To merge, we backfill (fill NaN with the next value in the col) the columns with lesser entries and create a df for each team."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/4270805705.py:8: DtypeWarning: Columns (1,2,3,4,5,6,7,8,9,10,11,12,14,15,17,18,20,22,23,25,27,29,31,33,35,37,39,41,43,44,45,47,49) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_tmp = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/{}/{}\".format(team, data))\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/4270805705.py:8: DtypeWarning: Columns (1,2,3,4,5,6,7,8,9,10,11,12,14,15,17,18,20,22,23,25,27,29,31,33,35,37,39,41,43,44,45,47,49) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_tmp = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/{}/{}\".format(team, data))\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/4270805705.py:8: DtypeWarning: Columns (1,2,3,4,5,6,7,8,9,10,11,12,14,15,17,18,20,22,23,25,27,29,31,33,35,37,39,41,43,44,45,47,49) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_tmp = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/{}/{}\".format(team, data))\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/4270805705.py:8: DtypeWarning: Columns (1,2,3,4,5,6,7,8,9,10,11,12,14,15,17,18,20,22,23,25,27,29,31,33,35,37,39,41,43,44,45,47,49) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_tmp = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/{}/{}\".format(team, data))\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/4270805705.py:8: DtypeWarning: Columns (1,2,3,4,5,6,7,8,9,10,11,12,14,15,17,18,20,22,23,25,27,29,31,33,35,37,39,41,43,44,45,47,49) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_tmp = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/{}/{}\".format(team, data))\n"
     ]
    }
   ],
   "source": [
    "teams = [\"Digilog\",\"AICU\",\"IUACAAS\",\"Automatoes\",\"TheAutomators\"]\n",
    "datasets = [\"GreenhouseClimate.csv\",\"Production.csv\",\"CropParameters.csv\",\"GrodanSens.csv\",\"LabAnalysis.csv\",\"Resources.csv\",\"TomQuality.csv\"]\n",
    "df_raw = []\n",
    "df_teams = []\n",
    "for team in teams:\n",
    "    df_team_raw = []\n",
    "    for data in datasets:\n",
    "        df_tmp = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/{}/{}\".format(team, data))\n",
    "        df_tmp = df_tmp.rename(columns={\"%Time\": \"%time\", \"%Time \": \"%time\"})  # rename Time col to have the same name \"%time\"\n",
    "        df_team_raw.append(df_tmp)\n",
    "\n",
    "    df_team = pd.concat(df_team_raw, sort=True, ignore_index=True)  # merge all the datasets\n",
    "    df_team = df_team.fillna(method=\"bfill\")  # bfill all na fields\n",
    "    df_team = df_team.fillna(method=\"ffill\")  # ffill all the rest of the fields\n",
    "    df_team = df_team.fillna(0)  # fill any remaining missing values with 0\n",
    "\n",
    "    df_teams.append(df_team)\n",
    "\n",
    "# Now, df_teams is a list of dataframes for each team with all datasets merged and missing values filled."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "%Juice                         0\n",
       "%time                          0\n",
       "Acid                           0\n",
       "AssimLight                     0\n",
       "Bite                           0\n",
       "                              ..\n",
       "water_sup                      0\n",
       "water_sup_intervals_sp_min     0\n",
       "water_sup_intervals_vip_min    0\n",
       "window_pos_lee_sp              0\n",
       "window_pos_lee_vip             0\n",
       "Length: 120, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_teams[0].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the merged team csv file\n",
    "for i in range(len(df_teams)):\n",
    "    df_teams[i].to_csv(\"team_{}.csv\".format(i),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observations, Actions and Rewards "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"%Juice\",\n",
      "\"%time\",\n",
      "\"Acid\",\n",
      "\"AssimLight\",\n",
      "\"Bite\",\n",
      "\"BlackScr\",\n",
      "\"CO2_cons\",\n",
      "\"CO2air\",\n",
      "\"Cum_irr\",\n",
      "\"Cum_trusses\",\n",
      "\"DMC_fruit\",\n",
      "\"Drain\",\n",
      "\"EC_drain_PC\",\n",
      "\"EC_slab1\",\n",
      "\"EC_slab2\",\n",
      "\"ElecHigh\",\n",
      "\"ElecLow\",\n",
      "\"EnScr\",\n",
      "\"Flavour \",\n",
      "\"Heat_cons\",\n",
      "\"HumDef\",\n",
      "\"Irr\",\n",
      "\"Nr_fruits_ClassA\",\n",
      "\"Nr_fruits_ClassB\",\n",
      "\"PipeGrow\",\n",
      "\"PipeLow\",\n",
      "\"ProdA\",\n",
      "\"ProdB\",\n",
      "\"Rhair\",\n",
      "\"Stem_elong\",\n",
      "\"Stem_thick\",\n",
      "\"TSS\",\n",
      "\"Tair\",\n",
      "\"Tot_PAR\",\n",
      "\"Tot_PAR_Lamps\",\n",
      "\"Truss development time \",\n",
      "\"VentLee\",\n",
      "\"Ventwind\",\n",
      "\"WC_slab1\",\n",
      "\"WC_slab2\",\n",
      "\"Weight\",\n",
      "\"Weight_fruits_ClassA\",\n",
      "\"Weight_fruits_ClassB\",\n",
      "\"assim_sp\",\n",
      "\"assim_vip\",\n",
      "\"avg_nr_harvested_trusses\",\n",
      "\"co2_dos\",\n",
      "\"co2_sp\",\n",
      "\"co2_vip\",\n",
      "\"drain_B\",\n",
      "\"drain_Ca\",\n",
      "\"drain_Cl\",\n",
      "\"drain_Cu\",\n",
      "\"drain_EC\",\n",
      "\"drain_Fe\",\n",
      "\"drain_HCO3\",\n",
      "\"drain_K\",\n",
      "\"drain_Mg\",\n",
      "\"drain_Mn\",\n",
      "\"drain_Mo\",\n",
      "\"drain_NH4\",\n",
      "\"drain_NO3\",\n",
      "\"drain_Na\",\n",
      "\"drain_PH\",\n",
      "\"drain_PO4\",\n",
      "\"drain_SO4\",\n",
      "\"drain_Si\",\n",
      "\"drain_Zn\",\n",
      "\"dx_sp\",\n",
      "\"dx_vip\",\n",
      "\"int_blue_sp\",\n",
      "\"int_blue_vip\",\n",
      "\"int_farred_sp\",\n",
      "\"int_farred_vip\",\n",
      "\"int_red_sp\",\n",
      "\"int_red_vip\",\n",
      "\"int_white_sp\",\n",
      "\"int_white_vip\",\n",
      "\"irr_B\",\n",
      "\"irr_Ca\",\n",
      "\"irr_Cl\",\n",
      "\"irr_Cu\",\n",
      "\"irr_EC\",\n",
      "\"irr_Fe\",\n",
      "\"irr_HCO3\",\n",
      "\"irr_K\",\n",
      "\"irr_Mg\",\n",
      "\"irr_Mn\",\n",
      "\"irr_Mo\",\n",
      "\"irr_NH4\",\n",
      "\"irr_NO3\",\n",
      "\"irr_Na\",\n",
      "\"irr_PH\",\n",
      "\"irr_PO4\",\n",
      "\"irr_SO4\",\n",
      "\"irr_Si\",\n",
      "\"irr_Zn\",\n",
      "\"pH_drain_PC\",\n",
      "\"plant_dens\",\n",
      "\"scr_blck_sp\",\n",
      "\"scr_blck_vip\",\n",
      "\"scr_enrg_sp\",\n",
      "\"scr_enrg_vip\",\n",
      "\"stem_dens \",\n",
      "\"t_grow_min_sp\",\n",
      "\"t_grow_min_vip\",\n",
      "\"t_heat_sp\",\n",
      "\"t_heat_vip\",\n",
      "\"t_rail_min_sp\",\n",
      "\"t_rail_min_vip\",\n",
      "\"t_slab1\",\n",
      "\"t_slab2\",\n",
      "\"t_vent_sp\",\n",
      "\"t_ventlee_vip\",\n",
      "\"t_ventwind_vip\",\n",
      "\"water_sup\",\n",
      "\"water_sup_intervals_sp_min\",\n",
      "\"water_sup_intervals_vip_min\",\n",
      "\"window_pos_lee_sp\",\n",
      "\"window_pos_lee_vip\",\n"
     ]
    }
   ],
   "source": [
    "for col in df_teams[0].columns:\n",
    "    print('\"{}\",'.format(col))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of observation, action and reward columns \n",
    "observation_cols = [\"%Juice\",\"%time\",\"Acid\",\"AssimLight\",\n",
    "                    \"Bite\",\"BlackScr\",\"CO2_cons\",\"CO2air\",\n",
    "                    \"Cum_irr\",\"Cum_trusses\",\"DMC_fruit\",\"Drain\",\n",
    "                    \"EC_drain_PC\",\"EC_slab1\",\"EC_slab2\",\"ElecHigh\",\n",
    "                    \"ElecLow\",\"EnScr\",\"Flavour \",\"Heat_cons\",\n",
    "                    \"HumDef\",\"Irr\",\"Nr_fruits_ClassA\",\"Nr_fruits_ClassB\",\n",
    "                    \"PipeGrow\",\"PipeLow\",\"ProdA\",\"ProdB\",\n",
    "                    \"Rhair\",\"Stem_elong\",\"Stem_thick\",\"TSS\",\n",
    "                    \"Tair\",\"Tot_PAR\",\"Tot_PAR_Lamps\",\"Truss development time \",\n",
    "                    \"VentLee\",\"Ventwind\",\"WC_slab1\",\"WC_slab2\",\n",
    "                    \"Weight\",\"Weight_fruits_ClassA\",\"Weight_fruits_ClassB\",\"avg_nr_harvested_trusses\",\n",
    "                    \"co2_dos\",\"water_sup\",\"t_slab1\",\"t_slab2\",\n",
    "                    \"pH_drain_PC\",\"plant_dens\",\"stem_dens \",]\n",
    "\n",
    "setpoint_columns = [\"assim_sp\",\"co2_sp\",\"dx_sp\",\"int_blue_sp\",\"int_farred_sp\", \\\n",
    "        \"int_red_sp\",\"int_white_sp\",\"scr_blck_sp\",\"scr_enrg_sp\",\"t_grow_min_sp\",\\\n",
    "        \"t_heat_sp\",\"t_rail_min_sp\",\"t_vent_sp\",\"water_sup_intervals_sp_min\",\\\n",
    "        \"window_pos_lee_sp\"]\n",
    "\n",
    "reward_cols = [\"Weight\",\"Weight_fruits_ClassA\",\"Weight_fruits_ClassB\",\"avg_nr_harvested_trusses\",\\\n",
    "              \"Nr_fruits_ClassA\",\"Nr_fruits_ClassB\",\"ProdA\",\"ProdB\",\n",
    "               \"%Juice\",\"Acid\",\"Bite\",\"Flavour \",\n",
    "               \"TSS\",\"Weight\",\"DMC_fruit\",]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns for team Digilog:\n",
      "Index(['%Juice', '%time', 'Acid', 'AssimLight', 'Bite', 'BlackScr', 'CO2_cons',\n",
      "       'CO2air', 'Cum_irr', 'Cum_trusses',\n",
      "       ...\n",
      "       't_slab1', 't_slab2', 't_vent_sp', 't_ventlee_vip', 't_ventwind_vip',\n",
      "       'water_sup', 'water_sup_intervals_sp_min',\n",
      "       'water_sup_intervals_vip_min', 'window_pos_lee_sp',\n",
      "       'window_pos_lee_vip'],\n",
      "      dtype='object', length=120)\n",
      "------------------------------\n",
      "Columns for team AICU:\n",
      "Index(['%Juice', '%time', 'Acid', 'AssimLight', 'Bite', 'BlackScr', 'CO2_cons',\n",
      "       'CO2air', 'Cum_irr', 'Cum_trusses',\n",
      "       ...\n",
      "       't_slab1', 't_slab2', 't_vent_sp', 't_ventlee_vip', 't_ventwind_vip',\n",
      "       'water_sup', 'water_sup_intervals_sp_min',\n",
      "       'water_sup_intervals_vip_min', 'window_pos_lee_sp',\n",
      "       'window_pos_lee_vip'],\n",
      "      dtype='object', length=120)\n",
      "------------------------------\n",
      "Columns for team IUACAAS:\n",
      "Index(['%Juice', '%time', 'Acid', 'AssimLight', 'Bite', 'BlackScr', 'CO2_cons',\n",
      "       'CO2air', 'Cum_irr', 'Cum_trusses',\n",
      "       ...\n",
      "       't_slab1', 't_slab2', 't_vent_sp', 't_ventlee_vip', 't_ventwind_vip',\n",
      "       'water_sup', 'water_sup_intervals_sp_min',\n",
      "       'water_sup_intervals_vip_min', 'window_pos_lee_sp',\n",
      "       'window_pos_lee_vip'],\n",
      "      dtype='object', length=125)\n",
      "------------------------------\n",
      "Columns for team Automatoes:\n",
      "Index(['%Juice', '%time', 'Acid', 'AssimLight', 'Bite', 'BlackScr', 'CO2_cons',\n",
      "       'CO2air', 'Cum_irr', 'Cum_trusses',\n",
      "       ...\n",
      "       't_slab1', 't_slab2', 't_vent_sp', 't_ventlee_vip', 't_ventwind_vip',\n",
      "       'water_sup', 'water_sup_intervals_sp_min',\n",
      "       'water_sup_intervals_vip_min', 'window_pos_lee_sp',\n",
      "       'window_pos_lee_vip'],\n",
      "      dtype='object', length=120)\n",
      "------------------------------\n",
      "Columns for team TheAutomators:\n",
      "Index(['%Juice', '%time', 'Acid', 'AssimLight', 'Bite', 'BlackScr', 'CO2_cons',\n",
      "       'CO2air', 'Cum_irr', 'Cum_trusses',\n",
      "       ...\n",
      "       't_slab1', 't_slab2', 't_vent_sp', 't_ventlee_vip', 't_ventwind_vip',\n",
      "       'water_sup', 'water_sup_intervals_sp_min',\n",
      "       'water_sup_intervals_vip_min', 'window_pos_lee_sp',\n",
      "       'window_pos_lee_vip'],\n",
      "      dtype='object', length=120)\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "for i, df_team in enumerate(df_teams):\n",
    "    print(f\"Columns for team {teams[i]}:\")\n",
    "    print(df_team.columns)\n",
    "    print(\"------------------------------\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[\"sum\"] # TODO: divide it with the max of all teams\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:6: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[\"sum\"] # TODO: divide it with the max of all teams\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[\"sum\"] # TODO: divide it with the max of all teams\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[\"sum\"] # TODO: divide it with the max of all teams\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
      "/var/folders/vx/1m1l_7hj50l31_jjvmr4z0hw0000gn/T/ipykernel_24316/1495549438.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_reward[\"sum\"] = df_reward[\"sum\"] # TODO: divide it with the max of all teams\n"
     ]
    }
   ],
   "source": [
    "# creating reward dataframes for each team\n",
    "df_rewards = []\n",
    "for df_team in df_teams:\n",
    "    df_reward = df_team[reward_cols+[\"%time\"]]\n",
    "    # accumulating the reward_cols to use it as reward\n",
    "    df_reward[\"sum\"] = df_reward[reward_cols].sum(axis=1)\n",
    "    # standardizing the rewards \n",
    "    df_reward[\"sum\"] = df_reward[\"sum\"] # TODO: divide it with the max of all teams \n",
    "    # appending the reward and time to df\n",
    "    df_rewards.append(df_reward[[\"%time\",\"sum\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              %time           sum\n",
      "count  95848.000000  95848.000000\n",
      "mean   43898.010621   1267.526337\n",
      "std       47.919487    787.790742\n",
      "min    43815.000000    477.920000\n",
      "25%    43856.513890    477.920000\n",
      "50%    43898.015625   2053.400000\n",
      "75%    43939.507810   2053.400000\n",
      "max    43981.000000   3283.290000\n",
      "              %time           sum\n",
      "count  95849.000000  95849.000000\n",
      "mean   43898.010381   2116.774022\n",
      "std       47.919295   1343.972378\n",
      "min    43815.000000    769.820000\n",
      "25%    43856.513890    769.820000\n",
      "50%    43898.013890   3458.010000\n",
      "75%    43939.506940   3458.010000\n",
      "max    43981.000000   3562.710000\n",
      "              %time           sum\n",
      "count  95848.000000  95848.000000\n",
      "mean   43898.010621   1509.021538\n",
      "std       47.919487   1124.844335\n",
      "min    43815.000000    381.610000\n",
      "25%    43856.513890    381.610000\n",
      "50%    43898.015625   2631.453200\n",
      "75%    43939.507810   2631.453200\n",
      "max    43981.000000   2760.713200\n",
      "              %time           sum\n",
      "count  95848.000000  95848.000000\n",
      "mean   43898.010621   1349.801116\n",
      "std       47.919487    610.065470\n",
      "min    43815.000000    738.330000\n",
      "25%    43856.513890    738.330000\n",
      "50%    43898.015625   1958.430000\n",
      "75%    43939.507810   1958.430000\n",
      "max    43981.000000   2810.480000\n",
      "              %time           sum\n",
      "count  95848.000000  95848.000000\n",
      "mean   43898.010621   1114.683488\n",
      "std       47.919487    460.970101\n",
      "min    43815.000000    652.707000\n",
      "25%    43856.513890    652.707000\n",
      "50%    43898.015625   1574.329000\n",
      "75%    43939.507810   1574.329000\n",
      "max    43981.000000   3416.894000\n"
     ]
    }
   ],
   "source": [
    "# understanding the reward distribution in each team\n",
    "for df_reward in df_rewards:\n",
    "    print(df_reward.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating observation dataframes for each teams\n",
    "df_observations = []\n",
    "for df_team in df_teams:\n",
    "    df_observations.append(df_team[observation_cols])\n",
    "    \n",
    "# creating actions dataframe for each teams \n",
    "df_actions = []\n",
    "for df_team in df_teams:\n",
    "    df_actions.append(df_team[setpoint_columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_values(team:int,index:int)->Dict[str,pd.Series]:\n",
    "    \"\"\"get_value(team,index) -> (observation_current,action,rewards,observation_next)\n",
    "    \"\"\"\n",
    "    observation_current = df_observations[team].iloc[index]\n",
    "    try:\n",
    "        observation_next = df_observations[team].iloc[index+1]\n",
    "    except:\n",
    "        observation_next = 0\n",
    "    \n",
    "    action = df_actions[team].iloc[index]\n",
    "    rewards = df_rewards[team].iloc[index,1] # get the sum col\n",
    "    return {\"observation_current\":observation_current, \"action\":action,\"rewards\":rewards,\"observation_next\":observation_next}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'observation_current': %Juice                          68.0\n",
       " %time                        43815.0\n",
       " Acid                            13.3\n",
       " AssimLight                         0\n",
       " Bite                           193.0\n",
       " BlackScr                          35\n",
       " CO2_cons                    0.006253\n",
       " CO2air                           474\n",
       " Cum_irr                          0.2\n",
       " Cum_trusses                      1.0\n",
       " DMC_fruit                       8.77\n",
       " Drain                            0.0\n",
       " EC_drain_PC                      1.9\n",
       " EC_slab1                         NaN\n",
       " EC_slab2                         NaN\n",
       " ElecHigh                    0.959833\n",
       " ElecLow                          0.0\n",
       " EnScr                             96\n",
       " Flavour                         78.0\n",
       " Heat_cons                     3.2838\n",
       " HumDef                           7.1\n",
       " Irr                              0.0\n",
       " Nr_fruits_ClassA                 0.0\n",
       " Nr_fruits_ClassB                 0.0\n",
       " PipeGrow                        71.0\n",
       " PipeLow                         58.4\n",
       " ProdA                           0.05\n",
       " ProdB                            0.1\n",
       " Rhair                           59.7\n",
       " Stem_elong                        15\n",
       " Stem_thick                       8.0\n",
       " TSS                              8.6\n",
       " Tair                            20.4\n",
       " Tot_PAR                            0\n",
       " Tot_PAR_Lamps                    0.0\n",
       " Truss development time          50.0\n",
       " VentLee                            1\n",
       " Ventwind                           0\n",
       " WC_slab1                         NaN\n",
       " WC_slab2                         NaN\n",
       " Weight                           9.5\n",
       " Weight_fruits_ClassA            70.0\n",
       " Weight_fruits_ClassB             0.0\n",
       " avg_nr_harvested_trusses         0.1\n",
       " co2_dos                          NaN\n",
       " water_sup                        2.0\n",
       " t_slab1                          NaN\n",
       " t_slab2                          NaN\n",
       " pH_drain_PC                      7.3\n",
       " plant_dens                       1.3\n",
       " stem_dens                        2.6\n",
       " Name: 0, dtype: object,\n",
       " 'action': assim_sp                        NaN\n",
       " co2_sp                          NaN\n",
       " dx_sp                           NaN\n",
       " int_blue_sp                     NaN\n",
       " int_farred_sp                   NaN\n",
       " int_red_sp                      NaN\n",
       " int_white_sp                    NaN\n",
       " scr_blck_sp                     NaN\n",
       " scr_enrg_sp                     NaN\n",
       " t_grow_min_sp                   NaN\n",
       " t_heat_sp                       NaN\n",
       " t_rail_min_sp                   NaN\n",
       " t_vent_sp                       NaN\n",
       " water_sup_intervals_sp_min      NaN\n",
       " window_pos_lee_sp               NaN\n",
       " Name: 0, dtype: object,\n",
       " 'rewards': 477.92,\n",
       " 'observation_next': %Juice                             68.0\n",
       " %time                       43815.00347\n",
       " Acid                               13.3\n",
       " AssimLight                            0\n",
       " Bite                              193.0\n",
       " BlackScr                             85\n",
       " CO2_cons                       0.006253\n",
       " CO2air                              470\n",
       " Cum_irr                             0.3\n",
       " Cum_trusses                         1.0\n",
       " DMC_fruit                          8.77\n",
       " Drain                               0.0\n",
       " EC_drain_PC                         1.9\n",
       " EC_slab1                            NaN\n",
       " EC_slab2                            NaN\n",
       " ElecHigh                       0.959833\n",
       " ElecLow                             0.0\n",
       " EnScr                                96\n",
       " Flavour                            78.0\n",
       " Heat_cons                        3.2838\n",
       " HumDef                              7.0\n",
       " Irr                                 0.0\n",
       " Nr_fruits_ClassA                    0.0\n",
       " Nr_fruits_ClassB                    0.0\n",
       " PipeGrow                           55.9\n",
       " PipeLow                            57.5\n",
       " ProdA                              0.05\n",
       " ProdB                               0.1\n",
       " Rhair                              60.8\n",
       " Stem_elong                           15\n",
       " Stem_thick                          8.0\n",
       " TSS                                 8.6\n",
       " Tair                               20.6\n",
       " Tot_PAR                               0\n",
       " Tot_PAR_Lamps                       0.0\n",
       " Truss development time             50.0\n",
       " VentLee                               1\n",
       " Ventwind                              0\n",
       " WC_slab1                            NaN\n",
       " WC_slab2                            NaN\n",
       " Weight                              9.5\n",
       " Weight_fruits_ClassA               70.0\n",
       " Weight_fruits_ClassB                0.0\n",
       " avg_nr_harvested_trusses            0.1\n",
       " co2_dos                          0.0005\n",
       " water_sup                           4.0\n",
       " t_slab1                             NaN\n",
       " t_slab2                             NaN\n",
       " pH_drain_PC                         7.3\n",
       " plant_dens                          1.3\n",
       " stem_dens                           2.6\n",
       " Name: 1, dtype: object}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_values(0,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>%Juice</th>\n",
       "      <th>%time</th>\n",
       "      <th>Acid</th>\n",
       "      <th>AssimLight</th>\n",
       "      <th>Bite</th>\n",
       "      <th>BlackScr</th>\n",
       "      <th>CO2_cons</th>\n",
       "      <th>CO2air</th>\n",
       "      <th>Cum_irr</th>\n",
       "      <th>Cum_trusses</th>\n",
       "      <th>...</th>\n",
       "      <th>Weight_fruits_ClassA</th>\n",
       "      <th>Weight_fruits_ClassB</th>\n",
       "      <th>avg_nr_harvested_trusses</th>\n",
       "      <th>co2_dos</th>\n",
       "      <th>water_sup</th>\n",
       "      <th>t_slab1</th>\n",
       "      <th>t_slab2</th>\n",
       "      <th>pH_drain_PC</th>\n",
       "      <th>plant_dens</th>\n",
       "      <th>stem_dens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>68.0</td>\n",
       "      <td>43815.00000</td>\n",
       "      <td>13.3</td>\n",
       "      <td>0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>35</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>474</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>2.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>68.0</td>\n",
       "      <td>43815.00347</td>\n",
       "      <td>13.3</td>\n",
       "      <td>0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>85</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>470</td>\n",
       "      <td>0.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>2.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>68.0</td>\n",
       "      <td>43815.00694</td>\n",
       "      <td>13.3</td>\n",
       "      <td>0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>96</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>482</td>\n",
       "      <td>0.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>2.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>68.0</td>\n",
       "      <td>43815.01042</td>\n",
       "      <td>13.3</td>\n",
       "      <td>0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>96</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>472</td>\n",
       "      <td>0.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>2.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>68.0</td>\n",
       "      <td>43815.01389</td>\n",
       "      <td>13.3</td>\n",
       "      <td>0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>96</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>469</td>\n",
       "      <td>0.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>2.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95843</th>\n",
       "      <td>60.0</td>\n",
       "      <td>43922.00000</td>\n",
       "      <td>14.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>263.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.032218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1291.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.3</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95844</th>\n",
       "      <td>55.0</td>\n",
       "      <td>43936.00000</td>\n",
       "      <td>14.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>298.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.032218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1291.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.3</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95845</th>\n",
       "      <td>68.0</td>\n",
       "      <td>43950.00000</td>\n",
       "      <td>12.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>190.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.032218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1291.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.3</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95846</th>\n",
       "      <td>69.0</td>\n",
       "      <td>43964.00000</td>\n",
       "      <td>12.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>148.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.032218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1291.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.3</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95847</th>\n",
       "      <td>57.0</td>\n",
       "      <td>43980.00000</td>\n",
       "      <td>9.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>182.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.032218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1291.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.3</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95848 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       %Juice        %time  Acid AssimLight   Bite BlackScr  CO2_cons CO2air  \\\n",
       "0        68.0  43815.00000  13.3          0  193.0       35  0.006253    474   \n",
       "1        68.0  43815.00347  13.3          0  193.0       85  0.006253    470   \n",
       "2        68.0  43815.00694  13.3          0  193.0       96  0.006253    482   \n",
       "3        68.0  43815.01042  13.3          0  193.0       96  0.006253    472   \n",
       "4        68.0  43815.01389  13.3          0  193.0       96  0.006253    469   \n",
       "...       ...          ...   ...        ...    ...      ...       ...    ...   \n",
       "95843    60.0  43922.00000  14.9        NaN  263.0      NaN  0.032218    NaN   \n",
       "95844    55.0  43936.00000  14.0        NaN  298.0      NaN  0.032218    NaN   \n",
       "95845    68.0  43950.00000  12.6        NaN  190.0      NaN  0.032218    NaN   \n",
       "95846    69.0  43964.00000  12.4        NaN  148.0      NaN  0.032218    NaN   \n",
       "95847    57.0  43980.00000   9.6        NaN  182.0      NaN  0.032218    NaN   \n",
       "\n",
       "      Cum_irr  Cum_trusses  ...  Weight_fruits_ClassA  Weight_fruits_ClassB  \\\n",
       "0         0.2          1.0  ...                  70.0                   0.0   \n",
       "1         0.3          1.0  ...                  70.0                   0.0   \n",
       "2         0.3          1.0  ...                  70.0                   0.0   \n",
       "3         0.3          1.0  ...                  70.0                   0.0   \n",
       "4         0.3          1.0  ...                  70.0                   0.0   \n",
       "...       ...          ...  ...                   ...                   ...   \n",
       "95843     NaN         22.0  ...                1291.0                 245.0   \n",
       "95844     NaN         22.0  ...                1291.0                 245.0   \n",
       "95845     NaN         22.0  ...                1291.0                 245.0   \n",
       "95846     NaN         22.0  ...                1291.0                 245.0   \n",
       "95847     NaN         22.0  ...                1291.0                 245.0   \n",
       "\n",
       "      avg_nr_harvested_trusses  co2_dos water_sup  t_slab1  t_slab2  \\\n",
       "0                          0.1      NaN       2.0      NaN      NaN   \n",
       "1                          0.1   0.0005       4.0      NaN      NaN   \n",
       "2                          0.1   0.0003       4.0      NaN      NaN   \n",
       "3                          0.1   0.0012       4.0      NaN      NaN   \n",
       "4                          0.1   0.0010       4.0      NaN      NaN   \n",
       "...                        ...      ...       ...      ...      ...   \n",
       "95843                      2.2      NaN       NaN      NaN      NaN   \n",
       "95844                      2.2      NaN       NaN      NaN      NaN   \n",
       "95845                      2.2      NaN       NaN      NaN      NaN   \n",
       "95846                      2.2      NaN       NaN      NaN      NaN   \n",
       "95847                      2.2      NaN       NaN      NaN      NaN   \n",
       "\n",
       "      pH_drain_PC  plant_dens  stem_dens   \n",
       "0             7.3         1.3         2.6  \n",
       "1             7.3         1.3         2.6  \n",
       "2             7.3         1.3         2.6  \n",
       "3             7.3         1.3         2.6  \n",
       "4             7.3         1.3         2.6  \n",
       "...           ...         ...         ...  \n",
       "95843         NaN         1.3         5.2  \n",
       "95844         NaN         1.3         5.2  \n",
       "95845         NaN         1.3         5.2  \n",
       "95846         NaN         1.3         5.2  \n",
       "95847         NaN         1.3         5.2  \n",
       "\n",
       "[95848 rows x 51 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_observations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_weather = pd.read_csv(\"/Users/faymajidelhassan/Downloads/AutonomousGreenhouseControl-main/archive/Weather/Weather.csv\")\n",
    "# give me \"%time\" i give weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>%time</th>\n",
       "      <th>AbsHumOut</th>\n",
       "      <th>Iglob</th>\n",
       "      <th>PARout</th>\n",
       "      <th>Pyrgeo</th>\n",
       "      <th>RadSum</th>\n",
       "      <th>Rain</th>\n",
       "      <th>Rhout</th>\n",
       "      <th>Tout</th>\n",
       "      <th>Winddir</th>\n",
       "      <th>Windsp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43815.00000</td>\n",
       "      <td>6.220954</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-72.0</td>\n",
       "      <td>215.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.6</td>\n",
       "      <td>6.9</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>43815.00347</td>\n",
       "      <td>6.220954</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-73.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.6</td>\n",
       "      <td>6.9</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>43815.00694</td>\n",
       "      <td>6.205565</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.4</td>\n",
       "      <td>6.9</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>43815.01042</td>\n",
       "      <td>6.190173</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-77.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.2</td>\n",
       "      <td>6.9</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43815.01389</td>\n",
       "      <td>6.162624</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-75.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.9</td>\n",
       "      <td>6.7</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47804</th>\n",
       "      <td>43980.98611</td>\n",
       "      <td>9.286397</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.999999e-01</td>\n",
       "      <td>-85.0</td>\n",
       "      <td>2992.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.4</td>\n",
       "      <td>15.1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47805</th>\n",
       "      <td>43980.98958</td>\n",
       "      <td>9.242139</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>-85.0</td>\n",
       "      <td>2992.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.5</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47806</th>\n",
       "      <td>43980.99306</td>\n",
       "      <td>9.152067</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.350000e-08</td>\n",
       "      <td>-84.0</td>\n",
       "      <td>2992.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70.8</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47807</th>\n",
       "      <td>43980.99653</td>\n",
       "      <td>9.177802</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-85.0</td>\n",
       "      <td>2992.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47808</th>\n",
       "      <td>43981.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>47809 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             %time  AbsHumOut  Iglob        PARout  Pyrgeo  RadSum  Rain  \\\n",
       "0      43815.00000   6.220954    0.0  0.000000e+00   -72.0   215.0   0.0   \n",
       "1      43815.00347   6.220954    0.0  0.000000e+00   -73.0     0.0   0.0   \n",
       "2      43815.00694   6.205565    0.0  0.000000e+00   -76.0     0.0   0.0   \n",
       "3      43815.01042   6.190173    0.0  0.000000e+00   -77.0     0.0   0.0   \n",
       "4      43815.01389   6.162624    0.0  0.000000e+00   -75.0     0.0   0.0   \n",
       "...            ...        ...    ...           ...     ...     ...   ...   \n",
       "47804  43980.98611   9.286397    0.0  9.999999e-01   -85.0  2992.0   0.0   \n",
       "47805  43980.98958   9.242139    0.0  1.000000e+00   -85.0  2992.0   0.0   \n",
       "47806  43980.99306   9.152067    0.0  3.350000e-08   -84.0  2992.0   0.0   \n",
       "47807  43980.99653   9.177802    0.0  0.000000e+00   -85.0  2992.0   0.0   \n",
       "47808  43981.00000        NaN    NaN           NaN     NaN     NaN   NaN   \n",
       "\n",
       "       Rhout  Tout  Winddir  Windsp  \n",
       "0       80.6   6.9     32.0     4.7  \n",
       "1       80.6   6.9     32.0     4.7  \n",
       "2       80.4   6.9     32.0     4.7  \n",
       "3       80.2   6.9     32.0     4.7  \n",
       "4       80.9   6.7     32.0     4.7  \n",
       "...      ...   ...      ...     ...  \n",
       "47804   71.4  15.1      2.0     4.3  \n",
       "47805   71.5  15.0      2.0     4.3  \n",
       "47806   70.8  15.0      2.0     3.8  \n",
       "47807   71.0  15.0      2.0     3.8  \n",
       "47808    NaN   NaN      NaN     NaN  \n",
       "\n",
       "[47809 rows x 11 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}