cem_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time step: 574"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-03f19716fc83>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     81\u001b[0m     \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m     action_next = agents.select_action(\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m     )  # passing in environment for Oracle agent.\n\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/research/RL/CityLearn/ROLEVT/agent.py\u001b[0m in \u001b[0;36mselect_action\u001b[0;34m(self, state, day_ahead)\u001b[0m\n\u001b[1;32m    345\u001b[0m         \u001b[0;34m\"\"\"Overrides from `TD3`. Utilizes CEM and Digital Twin computations\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    346\u001b[0m         \u001b[0;31m# update zeta\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 347\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_zeta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    348\u001b[0m         \u001b[0;31m# run forward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    349\u001b[0m         \u001b[0mactions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mday_ahead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/research/RL/CityLearn/ROLEVT/agent.py\u001b[0m in \u001b[0;36mset_zeta\u001b[0;34m(self, zeta)\u001b[0m\n\u001b[1;32m    327\u001b[0m         \u001b[0;34m\"\"\"Update zeta which will be supplied to `select_action`\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mzeta\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m             \u001b[0mzeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_zeta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# put into actor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_it\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrbc_threshold\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_it\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m24\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/research/RL/CityLearn/ROLEVT/agent.py\u001b[0m in \u001b[0;36mget_zeta\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0;31m# Getting the elite_set and elite_set_prev\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m         \u001b[0melite_set_eliteset_prev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_EliteSet_EliteSetPrev\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     98\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0melite_set_prev\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mk\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mK_keep\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/research/RL/CityLearn/ROLEVT/agent.py\u001b[0m in \u001b[0;36mset_EliteSet_EliteSetPrev\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    232\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mKbest\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m                     \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbest_zeta_args\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mKbest\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0melite_set\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mKbest\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melite_set_dummy\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0melite_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0melite_set\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# Run this again after editing submodules so Colab uses the updated versions\n",
    "from citylearn import  CityLearn\n",
    "from pathlib import Path\n",
    "from agent import Agent\n",
    "from copy import deepcopy\n",
    "import sys\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "import utils\n",
    "import time\n",
    "\n",
    "import numpy as np    \n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "# Load environment\n",
    "climate_zone = 5\n",
    "params = {'data_path':Path(\"data/Climate_Zone_\"+str(climate_zone)), \n",
    "        'building_attributes':'building_attributes.json', \n",
    "        'weather_file':'weather_data.csv', \n",
    "        'solar_profile':'solar_generation_1kW.csv', \n",
    "        'carbon_intensity':'carbon_intensity.csv',\n",
    "        'building_ids':[\"Building_\"+str(i) for i in [1,2,3,4,5,6,7,8,9]],\n",
    "        'buildings_states_actions':'buildings_state_action_space.json', \n",
    "        'simulation_period': (0, 8760*4-1), \n",
    "        'cost_function': ['ramping','1-load_factor','average_daily_peak','peak_demand','net_electricity_consumption','carbon_emissions'], \n",
    "        'central_agent': False,\n",
    "        'save_memory': False }\n",
    "\n",
    "# Contain the lower and upper bounds of the states and actions, to be provided to the agent to normalize the variables between 0 and 1.\n",
    "env = CityLearn(**params)\n",
    "observations_spaces, actions_spaces = env.get_state_action_spaces()\n",
    "\n",
    "# Provides information on Building type, Climate Zone, Annual DHW demand, Annual Cooling Demand, Annual Electricity Demand, Solar Capacity, and correllations among buildings\n",
    "building_info = env.get_building_information()\n",
    "\n",
    "\n",
    "params_agent = {'building_ids':[\"Building_\"+str(i) for i in [1,2,3,4,5,6,7,8,9]],\n",
    "                 'buildings_states_actions':'buildings_state_action_space.json', \n",
    "                 'building_info':building_info,\n",
    "                 'observation_spaces':observations_spaces, \n",
    "                 'action_spaces':actions_spaces}\n",
    "\n",
    "# Instantiating the control agent(s)\n",
    "agents = Agent(**params_agent)\n",
    "\n",
    "RBC_THRESHOLD = 24*14\n",
    "# agents = Agent(\n",
    "#     num_actions=actions_spaces,\n",
    "#     num_buildings=len(observations_spaces),\n",
    "#     env = env, is_oracle = False,\n",
    "# )\n",
    "\n",
    "state = env.reset()\n",
    "done = False\n",
    "\n",
    "action = agents.select_action(state, False)\n",
    "costs_peak_net_ele = []\n",
    "\n",
    "t_idx = 0\n",
    "# run for a month - NOTE: THIS WILL TAKE ~2 HOURS TO RUN. reduce `end_time` for quicker results.\n",
    "end_time = RBC_THRESHOLD + 24 * 300\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "\n",
    "\n",
    "# returns E_grid for RBC agent\n",
    "E_grid_RBC = utils.RBC(actions_spaces).get_rbc_data(\n",
    "    deepcopy(env), state, end_time)\n",
    "E_grid_RBC = np.array(E_grid_RBC)\n",
    "\n",
    "E_grid_true = []  # see comments below for more info.\n",
    "\n",
    "while not done and env.time_step < end_time:\n",
    "\n",
    "    \n",
    "\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    action_next = agents.select_action(\n",
    "        next_state, False\n",
    "    )  # passing in environment for Oracle agent.\n",
    "\n",
    "#     agents.add_to_buffer_oracle(state, env, action, reward, next_state)\n",
    "    agents.add_to_buffer(state, action, reward, next_state, done)\n",
    "    ## add env E-grid\n",
    "    E_grid_true.append([x[28] for x in state])\n",
    "    state = next_state\n",
    "    action = action_next\n",
    "\n",
    "    t_idx += 1\n",
    "\n",
    "    print(f\"\\rTime step: {t_idx}\", end=\"\")\n",
    "\n",
    "print(\n",
    "    f\"\\nTotal time (min) to run {end_time // 24} days of simulation: {round((time.time() - start_time) / 60, 3)}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.shape(agents.all_costs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vars_RL = agents.logger\n",
    "# print(len(vars_RL))\n",
    "# print(RBC_THRESHOLD)\n",
    "\n",
    "time_RBC = int(RBC_THRESHOLD/24)\n",
    "time_sim = int(end_time/24) - time_RBC + 1  # Number of days simulation\n",
    "\n",
    "\n",
    "# print(np.shape(agents.all_costs))\n",
    "\n",
    "all_costs = agents.all_costs\n",
    "\n",
    "all_costs = np.mean(all_costs, axis = 2)\n",
    "\n",
    "print(np.shape(all_costs))\n",
    "\n",
    "p_ele_data = np.array(agents.p_ele_logger)\n",
    "\n",
    "# print(np.shape(p_ele_data))\n",
    "# p_ele_data = agents.p_ele_logger.reshape(len(vars_RL)*1*24,9)\n",
    "# print(np.shape(agents.p_ele_logger))\n",
    "\n",
    "def heatmap2d(arr: np.ndarray):\n",
    "    plt.imshow(arr, cmap='viridis', aspect = 'auto')\n",
    "    plt.title('Heatmap Building 4')\n",
    "    plt.xlabel('days')\n",
    "    plt.ylabel('hours')\n",
    "    plt.colorbar()\n",
    "    plt.show()\n",
    "\n",
    "print(p_ele_data.shape)\n",
    "p_ele_data = p_ele_data[:,:,:,3]    # Getting data for a certain building \n",
    "\n",
    "p_ele_data = p_ele_data.reshape(time_sim, 24)\n",
    "\n",
    "plt.figure(figsize = (20,10))\n",
    "heatmap2d(p_ele_data[0:100,:].T)\n",
    "\n",
    "# print(p_ele_data[1,:])\n",
    "print(all_costs)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "############## TODO: ENTER CODE HERE ##############\n",
    "\n",
    "# list of dictionary of variables generated from RL. See actor.py#L166 for relevant variable names. eg. vars_RL[0]['E_grid']\n",
    "# NOTE: dimensions of RL/NORL logger is ≠ RBC. This is because RL/NORL implicitly uses RBC. We only start collecting data from\n",
    "# RL/NORL once it starts making inference, i.e, after `rbc_threshold`. Tweak the parameter above!\n",
    "\n",
    "\n",
    "vars_RL = agents.logger\n",
    "print(len(vars_RL))\n",
    "\n",
    "# list of dictionary of variables generated NORL - Optim w/o any RL. See actor.py#L166 for relevant variable names. eg. vars_RL[0]['E_grid']\n",
    "# vars_NORL = agents.norl_logger\n",
    "\n",
    "# true E-grid values. NOTE: E_grid = E_grid_true. E_grid_pred = var[\"E_grid\"] for RL/Optim\n",
    "E_grid_true = np.array(E_grid_true)\n",
    "\n",
    "# E_grid net electricity consumption per building using RBC\n",
    "E_grid_RBC = np.array(E_grid_RBC)\n",
    "\n",
    "############## TODO: ENTER CODE HERE ##############\n",
    "\n",
    "\n",
    "# vars_RL = np.array(vars_RL)  # DImension - No.of days\n",
    "# RL_SOC_C = np.flatten(vars_RL[:]['SOC_C']).reshape(24*30,9)\n",
    "# RL_SOC_C.append(vars_RL[i]['SOC_C'])\n",
    "\n",
    "# #If indexing error, use the for loop\n",
    "# RL_SOC_C.append( vars_RL[i][key] )\n",
    "# RL_SOC_C = np.array(RL_SOC_C).flatten().reshape(days * 24, 9)\n",
    "\n",
    "\n",
    "# vars_A, vars_B, ..., vars_Z = [], [], ..., [] # RL\n",
    "# vars_A, vars_B, ..., vars_Z = [], [], ..., [] # NORL  #Already Defined\n",
    "\n",
    "# 1. RBC Agent\n",
    "RBC_actions_arr = []\n",
    "RBC_look_ahead_cost = []  \n",
    "RBC_E_grid_pred = []\n",
    "RBC_E_grid_sell = []\n",
    "RBC_E_hpC = []\n",
    "RBC_E_ehH = []\n",
    "RBC_Edhw = []\n",
    "RBC_SOC_bat = []\n",
    "RBC_SOC_C = []\n",
    "RBC_SOC_H = []\n",
    "RBC_C_p_bat = []\n",
    "RBC_C_p_Csto = []\n",
    "RBC_C_p_Hsto = []\n",
    "RBC_ramping_cost = []\n",
    "RBC_peak_electricity_cost = []\n",
    "RBC_total_cost = []\n",
    "RBC_action_C = []\n",
    "RBC_action_H = []\n",
    "RBC_action_bat = []\n",
    "\n",
    "# 2. RL Agent\n",
    "RL_actions_arr = []\n",
    "RL_look_ahead_cost = [] \n",
    "RL_E_grid_sell = []\n",
    "RL_E_grid_pred = []\n",
    "RL_E_hpC = []\n",
    "RL_E_ehH = []\n",
    "RL_Edhw = []\n",
    "RL_SOC_bat = []\n",
    "RL_SOC_H = []\n",
    "RL_SOC_C = []\n",
    "RL_C_p_bat = []\n",
    "RL_C_p_Csto = []\n",
    "RL_C_p_Hsto = []\n",
    "RL_ramping_cost = []\n",
    "RL_peak_electricity_cost = []\n",
    "RL_total_cost = []\n",
    "RL_action_C = []\n",
    "RL_action_H = []\n",
    "RL_action_bat = []\n",
    "\n",
    "# 3. NO RL Agent (Just Optimization)\n",
    "# NORL_actions_arr = []\n",
    "# NORL_look_ahead_cost = []\n",
    "# NORL_E_grid_sell = []\n",
    "# Egrid = []  \n",
    "# NORL_E_grid_pred = []\n",
    "# NORL_E_hpC = []\n",
    "# NORL_E_ehH = []\n",
    "# NORL_Edhw = []\n",
    "# NORL_SOC_bat = []\n",
    "# NORL_SOC_C = []\n",
    "# NORL_SOC_H = []\n",
    "# NORL_C_p_bat = []\n",
    "# NORL_C_p_Csto = []\n",
    "# NORL_C_p_Hsto = []\n",
    "# NORL_ramping_cost = []\n",
    "# NORL_peak_electricity_cost = []\n",
    "# NORL_total_cost = []\n",
    "# NORL_action_C = []\n",
    "# NORL_action_H = []\n",
    "# NORL_action_bat = []\n",
    "\n",
    "\n",
    "# keys = list(\"E_grid\", \"E_grid_sell\", \"E_hpC\", \"E_ehH\", \"SOC_bat\", \"SOC_H\", \"SOC_C\", \" action_bat\", \"action_H\", \"action_C\") \n",
    "\n",
    "\n",
    "\n",
    "for i in range(len(vars_RL)): #number of days of RL/NORL\n",
    "    for j in range (9):\n",
    "        RL_E_grid_pred.append(vars_RL[i][j][\"E_grid\"])\n",
    "        RL_E_grid_sell.append(vars_RL[i][j][\"E_grid_sell\"])\n",
    "        RL_E_hpC.append(vars_RL[i][j][\"E_hpC\"])\n",
    "        RL_E_ehH.append(vars_RL[i][j]['E_ehH'])\n",
    "        RL_SOC_bat.append(vars_RL[i][j][\"SOC_bat\"])\n",
    "        RL_SOC_H.append(vars_RL[i][j][\"SOC_H\"])\n",
    "        RL_SOC_C.append(vars_RL[i][j][\"SOC_C\"])\n",
    "        RL_action_bat.append(vars_RL[i][j][\"action_bat\"])\n",
    "        RL_action_C.append(vars_RL[i][j][\"action_C\"])\n",
    "        RL_action_H.append(vars_RL[i][j][\"action_H\"])\n",
    "        \n",
    "    \n",
    "#         NORL_E_grid_pred.append(vars_NORL[i][j][\"E_grid\"])\n",
    "#         NORL_E_grid_sell.append(vars_NORL[i][j][\"E_grid_sell\"])\n",
    "#         NORL_E_hpC.append(vars_NORL[i][j][\"E_hpC\"])\n",
    "#         NORL_E_ehH.append(vars_NORL[i][j][\"E_ehH\"])\n",
    "#         NORL_SOC_bat.append(vars_NORL[i][j][\"SOC_bat\"])\n",
    "#         NORL_SOC_H.append(vars_NORL[i][j][\"SOC_H\"])\n",
    "#         NORL_SOC_C.append(vars_NORL[i][j][\"SOC_C\"])\n",
    "#         NORL_action_bat.append(vars_NORL[i][j][\"action_bat\"])\n",
    "#         NORL_action_C.append(vars_NORL[i][j][\"action_C\"])\n",
    "#         NORL_action_H.append(vars_NORL[i][j][\"action_H\"])\n",
    "\n",
    "\n",
    "\n",
    "# ### flatten out to get hour per building\n",
    "\n",
    "RL_E_grid_pred = np.array(RL_E_grid_pred).flatten().reshape(-1, 9) # hours x num_buildings\n",
    "RL_E_grid_sell = np.array(RL_E_grid_sell).flatten().reshape(-1, 9) # hours x num_buildings\n",
    "\n",
    "# RL_E_grid_pred = np.array(RL_E_grid_pred).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_E_grid_sell = np.array(RL_E_grid_sell).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_E_hpC = np.array(RL_E_hpC).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_E_ehH = np.array(RL_E_ehH).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_SOC_bat = np.array(RL_SOC_bat).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_SOC_H = np.array(RL_SOC_H).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_SOC_C = np.array(RL_SOC_C).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_action_bat = np.array(RL_action_bat).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_action_C = np.array(RL_action_C).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "# RL_action_H = np.array(RL_action_H).flatten().reshape(len(vars_RL), 9) # hours x num_buildings\n",
    "\n",
    "fig, axs = plt.subplots(3, 3, figsize=(15, 15))\n",
    "for i in range(3):\n",
    "    for j in range(3):\n",
    "        bid = i * 3 + j\n",
    "        axs[i, j].set_title(f\"Building {bid + 1}\")\n",
    "        axs[i, j].plot(\n",
    "            RL_E_grid_pred[:,i*3+j] + RL_E_grid_sell[:,i*3+j], label=\"CEM: E_grid\", marker='x'\n",
    "        )\n",
    "        axs[i, j].plot(\n",
    "            E_grid_true[RBC_THRESHOLD:,i*3+j], label=\"True E_grid\"\n",
    "        )  # plot true E grid\n",
    "        axs[i, j].plot(\n",
    "            E_grid_RBC[RBC_THRESHOLD:,i*3+j], label=\"RBC: E_grid\"\n",
    "        )  # plot true E grid\n",
    "        axs[i, j].grid()\n",
    "        if j == 0:\n",
    "            axs[i, j].set_ylabel(\"E grid\")\n",
    "        if i == 0:\n",
    "            axs[i, j].set_xlabel(\"Hour\")\n",
    "\n",
    "\n",
    "\n",
    "# # plot predicted E_grid\n",
    "week = end_time - 24 * 3  # plots last week of the month data\n",
    "# fig, axs = plt.subplots(3, 3, figsize=(15, 15))\n",
    "# for i in range(3):\n",
    "#     for j in range(3):\n",
    "#         bid = i * 3 + j\n",
    "# #         axs[i, j].set_title(f\"Building {bid + 1}\")\n",
    "# #         axs[i, j].plot(\n",
    "# #             NORL_E_grid_pred[:,i*3+j],\n",
    "# #             label=\"NORL Egrid\",\n",
    "# #         )   # plots per month\n",
    "#         axs[i, j].plot(\n",
    "#              E_grid_RBC[:,i*3+j], label=\"RBC E grid\"\n",
    "#          )  # plot true E grid\n",
    "#         axs[i, j].plot(\n",
    "#              RL_E_grid_pred[:,i*3+j], label=\"CEM RL_Egrid\"\n",
    "#          )  \n",
    "#         axs[i, j].grid()\n",
    "#         if j == 0:\n",
    "#             axs[i, j].set_ylabel(\"E grid\")\n",
    "#         if i == 0:\n",
    "#             axs[i, j].set_xlabel(\"Hour\")\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "time_RBC = int(RBC_THRESHOLD/24)\n",
    "time_sim = int(end_time/24) - time_RBC  # Number of days simulation\n",
    "time_end = int(end_time/24)\n",
    "\n",
    "ramping_cost_CEM = []\n",
    "ramping_cost_RBC = []\n",
    "peak_electricity_cost_CEM = []\n",
    "peak_electricity_cost_RBC = []\n",
    "\n",
    "a = np.shape(E_grid_RBC)\n",
    "print(a)\n",
    "b = np.shape(RL_E_grid_pred)\n",
    "print(b)\n",
    "\n",
    "E_grid_RBC_used = E_grid_RBC[RBC_THRESHOLD: end_time,:]\n",
    "RL_E_grid_pred_used = RL_E_grid_pred[0:24*time_sim,:]\n",
    "\n",
    "E_grid_RBC_used = E_grid_RBC_used.flatten().reshape(time_sim, 24, 9)\n",
    "RL_E_grid_pred_used = RL_E_grid_pred_used.flatten().reshape(time_sim,24, 9)\n",
    "\n",
    "\n",
    "for i in range(time_sim):\n",
    "#     t_start = week + i * 24\n",
    "#     t_end = week + (i + 1) * 24\n",
    "    ramping_cost_CEM_t = []\n",
    "    ramping_cost_RBC_t = []\n",
    "    peak_electricity_cost_CEM_t = []\n",
    "    peak_electricity_cost_RBC_t = []\n",
    "    for bid in range(9):\n",
    "        CEM_E_grid_t = RL_E_grid_pred_used[i,:,bid]\n",
    "        RBC_Egrid_t = E_grid_RBC_used[i,:,bid]\n",
    "        ramping_cost_CEM_t.append(np.sum(np.abs(CEM_E_grid_t[1:] - CEM_E_grid_t[:-1])))\n",
    "        ramping_cost_RBC_t.append(np.sum(np.abs(RBC_Egrid_t[1:] - RBC_Egrid_t[:-1])))\n",
    "        peak_electricity_cost_CEM_t.append(np.max(CEM_E_grid_t))\n",
    "        peak_electricity_cost_RBC_t.append(np.max(RBC_Egrid_t))\n",
    "    ramping_cost_CEM.append(ramping_cost_CEM_t)\n",
    "    ramping_cost_RBC.append(ramping_cost_RBC_t)\n",
    "    peak_electricity_cost_CEM.append(peak_electricity_cost_CEM_t)\n",
    "    peak_electricity_cost_RBC.append(peak_electricity_cost_RBC_t)\n",
    "\n",
    "\n",
    "CEM_cost = {\n",
    "    \"ramping_cost\": np.array(ramping_cost_CEM).T,\n",
    "    \"peak_electricity_cost\": np.array(peak_electricity_cost_CEM).T,\n",
    "    \"total_cost\": np.array(ramping_cost_CEM).T\n",
    "    + np.array(peak_electricity_cost_CEM).T,\n",
    "}\n",
    "RBC_cost = {\n",
    "    \"ramping_cost\": np.array(ramping_cost_RBC).T,\n",
    "    \"peak_electricity_cost\": np.array(peak_electricity_cost_RBC).T,\n",
    "    \"total_cost\": np.array(ramping_cost_RBC).T + np.array(peak_electricity_cost_RBC).T,\n",
    "}\n",
    "\n",
    "item_cost = [\"ramping_cost\", \"peak_electricity_cost\", \"total_cost\"]\n",
    "# for k in range(len(item_cost)):\n",
    "#     fig, axs = plt.subplots(3, 3, figsize=(15, 15))\n",
    "#     for i in range(3):\n",
    "#         for j in range(3):\n",
    "#             bid = i * 3 + j\n",
    "#             axs[i, j].set_title(f\"Building {bid + 1}: {item_cost[k]}\")\n",
    "#             axs[i, j].plot(\n",
    "#                 CEM_cost[item_cost[k]][bid, :], label=f\"CEM: {item_cost[k]}\"\n",
    "#             )  # plot true E grid\n",
    "#             axs[i, j].plot(RBC_cost[item_cost[k]][bid, :], label=f\"RBC: {item_cost[k]}\")\n",
    "#             axs[i, j].grid()\n",
    "#             if j == 0:\n",
    "#                 axs[i, j].set_ylabel(\"Cost\")\n",
    "#             if i == 0:\n",
    "#                 axs[i, j].set_xlabel(\"Day\")\n",
    "#     plt.legend()\n",
    "\n",
    "# for k in range(len(item_cost)):\n",
    "# fig, axs = plt.subplots(3, 3, figsize=(15, 15))\n",
    "# for i in range(3):\n",
    "#     for j in range(3):\n",
    "#         bid = i * 3 + j\n",
    "#         axs[i, j].set_title(f\"Building {bid + 1}: total cost CEM/RBC\")\n",
    "#         axs[i, j].plot(\n",
    "#             CEM_cost[\"total_cost\"][bid, :] / RBC_cost[\"total_cost\"][bid, :],\n",
    "#             label=f\"CEM/RBC\",\n",
    "#         )  # plot true E grid\n",
    "#         axs[i, j].grid()\n",
    "#         if j == 0:\n",
    "#             axs[i, j].set_ylabel(\"Cost (Ratio)\")\n",
    "#         if i == 0:\n",
    "#                 axs[i, j].set_xlabel(\"Day\")\n",
    "# plt.legend()\n",
    "\n",
    "print(all_costs)\n",
    "\n",
    "fig1, ax1 = plt.subplots()\n",
    "ax1.set_title(f\"Total cost CEM/RBC\")\n",
    "ax1.plot(np.sum(CEM_cost[\"total_cost\"],axis=0) / np.sum(RBC_cost[\"total_cost\"],axis=0),\n",
    "            label=f\"CEM/RBC\")  # plot true E grid\n",
    "ax1.plot(all_costs, label=f\"CEM daily costs\")\n",
    "ax1.grid()\n",
    "ax1.set_ylabel(\"Cost (Ratio)\")\n",
    "ax1.set_xlabel(\"Day\")\n",
    "plt.legend()\n",
    "\n",
    "print(np.mean(np.sum(CEM_cost[\"total_cost\"],axis=0) / np.sum(RBC_cost[\"total_cost\"],axis=0)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vars_RL = agents.logger\n",
    "# print(len(vars_RL))\n",
    "\n",
    "mean_elite_set = np.array(agents.mean_elite_set)\n",
    "\n",
    "# print(np.shape(agents.mean_elite_set))\n",
    "\n",
    "# p_ele_data = agents.p_ele_logger.reshape(len(vars_RL)*1*24,9)\n",
    "\n",
    "# print(np.shape(agents.p_ele_logger))\n",
    "\n",
    "fig, axs = plt.subplots(3, 3, figsize=(15, 15))\n",
    "for i in range(3):\n",
    "    for j in range(3):\n",
    "        bid = i * 3 + j\n",
    "        axs[i, j].set_title(f\"Building {bid + 1}\")\n",
    "        axs[i, j].plot(\n",
    "            mean_elite_set[:,i*3+j],\n",
    "            label=\"mean_elite_set\",\n",
    "        )   # plots per month\n",
    "#         axs[i, j].plot(\n",
    "#              E_grid_RBC[:,i*3+j], label=\"RBC E grid\"\n",
    "#          )  # plot true E grid\n",
    "#         axs[i, j].plot(\n",
    "#              RL_E_grid_pred[:,i*3+j], label=\"CEM RL_Egrid\"\n",
    "#          )  \n",
    "        axs[i, j].grid()\n",
    "        if j == 0:\n",
    "            axs[i, j].set_ylabel(\"mean_elite_set\")\n",
    "        if i == 0:\n",
    "            axs[i, j].set_xlabel(\"Day\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "g1 = np.sum(CEM_cost[\"total_cost\"],axis=0) /np.sum(RBC_cost[\"total_cost\"],axis=0)\n",
    "g1 = g1[0:365]\n",
    "g2 = all_costs\n",
    "\n",
    "\n",
    "plt.scatter(g1, g2)\n",
    "plt.title('Scatter plot')\n",
    "plt.xlabel('CEM/RBC')\n",
    "plt.ylabel('CEM daily costs')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# list_data = list(range(366))\n",
    "# list_data = tuple(list_data)\n",
    "\n",
    "# colors = (\"red\", \"green\")\n",
    "# groups = (\"CEM/RBC\", \"CEM daily costs\")\n",
    "\n",
    "# # Create plot\n",
    "# fig = plt.figure()\n",
    "# ax = fig.add_subplot(1, 1, 1, facecolor=\"1.0\")\n",
    "\n",
    "\n",
    "\n",
    "# for i, color, group in zip(list_data, colors, groups):\n",
    "#     x = g1[i]\n",
    "#     y = g2[i,:]\n",
    "#     ax.scatter(x, y, alpha=0.8, c=color, edgecolors='none', s=30, label=group)\n",
    "\n",
    "\n",
    "# plt.title('Matplot scatter plot')\n",
    "# plt.legend(loc=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.shape(all_costs))\n",
    "print(np.shape(np.sum(CEM_cost[\"total_cost\"],axis=0) / np.sum(RBC_cost[\"total_cost\"],axis=0)))\n",
    "A = np.sum(CEM_cost[\"total_cost\"],axis=0) /np.sum(RBC_cost[\"total_cost\"],axis=0)\n",
    "print(np.shape(A[0:365]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_RBC = int(RBC_THRESHOLD/24)\n",
    "time_sim = int(end_time/24) - time_RBC  # Number of days simulation\n",
    "\n",
    "time_end = int(end_time/24)\n",
    "\n",
    "print(time_sim)\n",
    "print(time_RBC)\n",
    "print(time_end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "A = np.ones((1,9))\n",
    "B = np.ones((1,9))\n",
    "\n",
    "C = np.vstack((A,B))\n",
    "\n",
    "# D = np.vstack()\n",
    "\n",
    "print(np.shape(C))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = [np.ones(((1,24,9))), np.ones(((1,24,9))), np.ones(((1,24,9)))]\n",
    "\n",
    "A = np.vstack(A)\n",
    "\n",
    "print(np.shape(A))\n",
    "\n",
    "print(type(A))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}