Added sparse discrete mean reverting oracle and config.

README.md

@@ -18,4 +18,5 @@ cd project
 git clone https://github.com/abides-sim/abides.git
 cd abides
 pip install -r requirements.txt
-```
+```
+

agent/ExchangeAgent.py

@@ -10,7 +10,6 @@
 from util.util import log_print
 
 import jsons as js
-import numpy as np
 import pandas as pd
 pd.set_option('display.max_rows', 500)
 
@@ -78,13 +77,18 @@ def kernelInitializing (self, kernel):
   def kernelTerminating (self):
     super().kernelTerminating()
 
-    if self.book_freq is None:
-      for symbol in self.order_books:
-        book = self.order_books[symbol]
-        dfLog = pd.DataFrame([book.mid_dict, book.bid_levels_price_dict, book.bid_levels_size_dict,
-                              book.ask_levels_price_dict, book.ask_levels_size_dict]).T
-        dfLog.columns = ['mid_price', 'bid_level_prices', 'bid_level_sizes', 'ask_level_prices', 'ask_level_sizes']
-        self.writeLog(dfLog, filename='orderbook_{}'.format(symbol))
+    # If the oracle supports writing the fundamental value series for its
+    # symbols, write them to disk.
+    if hasattr(self.oracle, 'f_log'):
+      for symbol in self.oracle.f_log:
+        dfFund = pd.DataFrame(self.oracle.f_log[symbol])
+        dfFund.set_index('FundamentalTime', inplace=True)
+        self.writeLog(dfFund, filename='fundamental_{}'.format(symbol))
+
+      print ("Fundamental archival complete.")
+
+    # Skip order book dump if requested.
+    if self.book_freq is None: return
 
     # Iterate over the order books controlled by this exchange.
     for symbol in self.order_books:
@@ -121,10 +125,7 @@ def kernelTerminating (self):
         quotes = sorted(dfLog.index.get_level_values(1).unique())
         min_quote = quotes[0]
         max_quote = quotes[-1]
-        try:
-          quotes = range(min_quote, max_quote+1)
-        except Exception as e:
-          quotes = np.arange(min_quote, max_quote + 0.01, step=0.01)
+        quotes = range(min_quote, max_quote+1)
 
         # Restructure the log to have multi-level rows of all possible pairs of time and quote
         # with volume as the only column.
@@ -138,7 +139,6 @@ def kernelTerminating (self):
         df['Volume'] = dfLog
 
 
-
         # Archive the order book snapshots directly to a file named with the symbol, rather than
         # to the exchange agent log.
         self.writeLog(df, filename='orderbook_{}'.format(symbol))
@@ -270,6 +270,12 @@ def receiveMessage (self, currentTime, msg):
         # Hand the order to the order book for processing.
         self.order_books[order.symbol].cancelOrder(deepcopy(order))
     elif msg.body['msg'] == 'MODIFY_ORDER':
+      # Replace an existing order with a modified order.  There could be some timing issues
+      # here.  What if an order is partially executed, but the submitting agent has not
+      # yet received the norification, and submits a modification to the quantity of the
+      # (already partially executed) order?  I guess it is okay if we just think of this
+      # as "delete and then add new" and make it the agent's problem if anything weird
+      # happens.
       order = msg.body['order']
       new_order = msg.body['new_order']
       log_print ("{} received MODIFY_ORDER: {}, new order: {}".format(self.name, order, new_order))

agent/ExperimentalAgent.py

@@ -24,8 +24,8 @@ def wakeup(self, currentTime):
             return
         elif (currentTime > self.mkt_open) and (currentTime < self.mkt_close):
             if currentTime == self.execution_timestamp:
-                self.placeLimitOrder(self.symbol, self.quantity, self.is_buy_order, self.limit_price, dollar=False)
-                if self.log_orders: self.logEvent('LIMIT_ORDER', {'agent_id': self.id, 'dollar': False, 'fill_price': None,
+                self.placeLimitOrder(self.symbol, self.quantity, self.is_buy_order, self.limit_price)
+                if self.log_orders: self.logEvent('LIMIT_ORDER', {'agent_id': self.id, 'fill_price': None,
                                                                   'is_buy_order': self.is_buy_order, 'limit_price': self.limit_price,
                                                                   'order_id': 1, 'quantity': self.quantity, 'symbol': self.symbol,
                                                                   'time_placed': str(currentTime)})

agent/MarketReplayAgent.py

@@ -58,14 +58,13 @@ def placeOrder(self, currentTime, order):
         existing_order = self.orders.get(id)
 
         if type == 'NEW':
-            self.placeLimitOrder(self.symbol, vol, direction == 'BUY', float(price), dollar=False, order_id=id)
+            self.placeLimitOrder(self.symbol, vol, direction == 'BUY', int(price), order_id=id)
         elif type in ['CANCELLATION', 'PARTIAL_CANCELLATION']:
             if existing_order:
                 if type == 'CANCELLATION':
                     self.cancelOrder(existing_order)
                 elif type == 'PARTIAL_CANCELLATION':
-                    new_order = LimitOrder(self.id, currentTime, self.symbol, vol, direction == 'BUY', float(price),
-                                           dollar=False, order_id=id)
+                    new_order = LimitOrder(self.id, currentTime, self.symbol, vol, direction == 'BUY', int(price), order_id=id)
                     self.modifyOrder(existing_order, new_order)
         elif type in ['EXECUTE_VISIBLE', 'EXECUTE_HIDDEN']:
             if existing_order:
@@ -76,8 +75,7 @@ def placeOrder(self, currentTime, order):
                     if new_vol == 0:
                         self.cancelOrder(existing_order)
                     else:
-                        executed_order = LimitOrder(self.id, currentTime, self.symbol, new_vol, direction == 'BUY', float(price),
-                                                    dollar=False, order_id=id)
+                        executed_order = LimitOrder(self.id, currentTime, self.symbol, new_vol, direction == 'BUY', int(price), order_id=id)
                         self.modifyOrder(existing_order, executed_order)
                         self.orders.get(id).quantity = new_vol
     else:

agent/TradingAgent.py

@@ -253,8 +253,10 @@ def getOrderStream (self, symbol, length=1):
 
   # Used by any Trading Agent subclass to place a limit order.  Parameters expect:
   # string (valid symbol), int (positive share quantity), bool (True == BUY), int (price in cents).
-  def placeLimitOrder (self, symbol, quantity, is_buy_order, limit_price, dollar=True, order_id=None, ignore_risk = False):
-    order = LimitOrder(self.id, self.currentTime, symbol, quantity, is_buy_order, limit_price, dollar, order_id)
+  # The call may optionally specify an order_id (otherwise global autoincrement is used) and
+  # whether cash or risk limits should be enforced or ignored for the order.
+  def placeLimitOrder (self, symbol, quantity, is_buy_order, limit_price, order_id=None, ignore_risk = True):
+    order = LimitOrder(self.id, self.currentTime, symbol, quantity, is_buy_order, limit_price, order_id)
 
     if quantity > 0:
       # Test if this order can be permitted given our at-risk limits.
@@ -300,7 +302,8 @@ def cancelOrder (self, order):
     if self.log_orders: self.logEvent('CANCEL_SUBMITTED', js.dump(order))
 
   # Used by any Trading Agent subclass to modify any existing limitorder.  The order must currently
-  # appear in the agent's open orders list.
+  # appear in the agent's open orders list.  Some additional tests might be useful here
+  # to ensure the old and new orders are the same in some way.
   def modifyOrder (self, order, newOrder):
     self.sendMessage(self.exchangeID, Message({ "msg" : "MODIFY_ORDER", "sender": self.id,
                                                 "order" : order, "new_order" : newOrder}))

agent/ZeroIntelligenceAgent.py

@@ -220,7 +220,14 @@ def updateEstimates (self):
     # Now having a best estimate of the fundamental at time t, we can make our best estimate
     # of the final fundamental (for time T) as of current time t.  Delta is now the number
     # of time steps remaining until the simulated exchange closes.
-    delta = (self.mkt_close - self.currentTime) / np.timedelta64(1, 'ns')
+    delta = max(0, (self.mkt_close - self.currentTime) / np.timedelta64(1, 'ns'))
+
+    # IDEA: instead of letting agent "imagine time forward" to the end of the day,
+    #       impose a maximum forward delta, like ten minutes or so.  This could make
+    #       them think more like traders and less like long-term investors.  Add
+    #       this line of code (keeping the max() line above) to try it.
+    #delta = min(delta, 1000000000 * 60 * 10)
+
     r_T  = (1 - (1 - self.kappa) ** delta) * self.r_bar
     r_T += ((1 - self.kappa) ** delta) * r_tprime
 

cli/sparse_fundamental.py

@@ -0,0 +1,76 @@
+import ast
+import matplotlib
+matplotlib.use('TkAgg')
+import matplotlib.pyplot as plt
+import pandas as pd
+import os
+import re
+import sys
+
+from joblib import Memory
+
+# Auto-detect terminal width.
+pd.options.display.width = None
+pd.options.display.max_rows = 1000
+pd.options.display.max_colwidth = 200
+
+BETWEEN_START = pd.to_datetime('09:30').time()
+BETWEEN_END = pd.to_datetime('16:00:00').time()
+
+# Linewidth for plots.
+LW = 2
+
+# Main program starts here.
+
+if len(sys.argv) < 2:
+  print ("Usage: python sparse_fundamental.py <Simulator Fundamental file>")
+  sys.exit()
+
+# TODO: only really works for one symbol right now.
+sim_file = sys.argv[1]
+
+m = re.search(r'fundamental_(.+?)\.', sim_file)
+
+if not m:
+  print ("Usage: python sparse_fundamental.py <Simulator Fundamental file>")
+  print ("{} does not appear to be a fundamental value log.".format(sim_file))
+  print ()
+  sys.exit()
+
+symbol = m.group(1)
+
+print ("Visualizing simulated fundamental from {}".format(sim_file))
+df_sim = pd.read_pickle(sim_file, compression='bz2')
+
+plt.rcParams.update({'font.size': 12})
+
+print (df_sim.head())
+
+# Use to restrict time to plot.
+#df_sim = df_sim.between_time(BETWEEN_START, BETWEEN_END)
+
+fig,ax = plt.subplots(figsize=(12,9), nrows=1, ncols=1)
+axes = [ax]
+
+# For smoothing...
+#hist_window = 100
+#sim_window = 100
+
+hist_window = 1
+sim_window = 1
+
+#df_sim['PRICE'] = df_sim['PRICE'].rolling(window=sim_window).mean()
+
+df_sim['FundamentalValue'].plot(color='C1', grid=True, linewidth=LW, alpha=0.9, ax=axes[0])
+axes[0].legend(['Simulated'])
+
+
+plt.suptitle('Fundamental Value: {}'.format(symbol))
+
+axes[0].set_ylabel('Fundamental Value')
+axes[0].set_xlabel('Fundamental Time')
+
+#plt.savefig('background_{}.png'.format(b))
+
+plt.show()
+

cli/sparse_midpoint.py

@@ -0,0 +1,157 @@
+import ast
+import matplotlib
+matplotlib.use('TkAgg')
+import matplotlib.pyplot as plt
+import pandas as pd
+import os
+import sys
+
+from joblib import Memory
+
+# Auto-detect terminal width.
+pd.options.display.width = None
+pd.options.display.max_rows = 1000
+pd.options.display.max_colwidth = 200
+
+# Initialize a persistent memcache.
+mem_hist = Memory(cachedir='./.cached_plot_hist', verbose=0)
+mem_sim = Memory(cachedir='./.cached_plot_sim', verbose=0)
+
+
+PRINT_BASELINE = False
+PRINT_DELTA_ONLY = False
+
+BETWEEN_START = pd.to_datetime('09:30').time()
+BETWEEN_END = pd.to_datetime('16:00:00').time()
+
+# Linewidth for plots.
+LW = 2
+
+# Used to read and cache simulated quotes.
+# Doesn't actually pay attention to symbols yet.
+#@mem_sim.cache
+def read_simulated_quotes (file, symbol):
+  print ("Simulated quotes were not cached.  This will take a minute.")
+  df = pd.read_pickle(file, compression='bz2')
+  df['Timestamp'] = df.index
+
+  # Keep only the last bid and last ask event at each timestamp.
+  df = df.drop_duplicates(subset=['Timestamp','EventType'], keep='last')
+
+  del df['Timestamp']
+
+  df_bid = df[df['EventType'] == 'BEST_BID'].copy()
+  df_ask = df[df['EventType'] == 'BEST_ASK'].copy()
+
+  if len(df) <= 0:
+    print ("There appear to be no simulated quotes.")
+    sys.exit()
+
+  df_bid['BEST_BID'] = [b for s,b,bv in df_bid['Event'].str.split(',')]
+  df_bid['BEST_BID_VOL'] = [bv for s,b,bv in df_bid['Event'].str.split(',')]
+  df_ask['BEST_ASK'] = [a for s,a,av in df_ask['Event'].str.split(',')]
+  df_ask['BEST_ASK_VOL'] = [av for s,a,av in df_ask['Event'].str.split(',')]
+
+  df_bid['BEST_BID'] = df_bid['BEST_BID'].str.replace('$','').astype('float64')
+  df_ask['BEST_ASK'] = df_ask['BEST_ASK'].str.replace('$','').astype('float64')
+
+  df_bid['BEST_BID_VOL'] = df_bid['BEST_BID_VOL'].astype('float64')
+  df_ask['BEST_ASK_VOL'] = df_ask['BEST_ASK_VOL'].astype('float64')
+
+  df = df_bid.join(df_ask, how='outer', lsuffix='.bid', rsuffix='.ask')
+  df['BEST_BID'] = df['BEST_BID'].ffill().bfill()
+  df['BEST_ASK'] = df['BEST_ASK'].ffill().bfill()
+  df['BEST_BID_VOL'] = df['BEST_BID_VOL'].ffill().bfill()
+  df['BEST_ASK_VOL'] = df['BEST_ASK_VOL'].ffill().bfill()
+
+  df['MIDPOINT'] = (df['BEST_BID'] + df['BEST_ASK']) / 2.0
+
+  return df
+
+
+
+# Main program starts here.
+
+if len(sys.argv) < 3:
+  print ("Usage: python midpoint_plot.py <Ticker symbol> <Simulator DataFrame file>")
+  sys.exit()
+
+# TODO: only really works for one symbol right now.
+
+symbol = sys.argv[1]
+sim_file = sys.argv[2]
+
+print ("Visualizing simulated {} from {}".format(symbol, sim_file))
+df_sim = read_simulated_quotes(sim_file, symbol)
+
+if PRINT_BASELINE:
+  baseline_file = os.path.join(os.path.dirname(sim_file) + '_baseline', os.path.basename(sim_file))
+  print (baseline_file)
+  df_baseline = read_simulated_quotes(baseline_file, symbol)
+
+plt.rcParams.update({'font.size': 12})
+
+
+
+# Use to restrict time to plot.
+df_sim = df_sim.between_time(BETWEEN_START, BETWEEN_END)
+
+if PRINT_BASELINE:
+  df_baseline = df_baseline.between_time(BETWEEN_START, BETWEEN_END)
+
+fig,ax = plt.subplots(figsize=(12,9), nrows=1, ncols=1)
+axes = [ax]
+
+# For smoothing...
+#hist_window = 100
+#sim_window = 100
+
+hist_window = 1
+sim_window = 1
+
+if PRINT_BASELINE:
+  # For nanosecond experiments, turn it into int index.  Pandas gets weird if all
+  # the times vary only by a few nanoseconds.
+  rng = pd.date_range(start=df_sim.index[0], end=df_sim.index[-1], freq='1N')
+
+  df_baseline = df_baseline[~df_baseline.index.duplicated(keep='last')]
+  df_baseline = df_baseline.reindex(rng,method='ffill')
+  df_baseline = df_baseline.reset_index(drop=True)
+
+  df_sim = df_sim[~df_sim.index.duplicated(keep='last')]
+  df_sim = df_sim.reindex(rng,method='ffill')
+  df_sim = df_sim.reset_index(drop=True)
+
+  # Print both separately.
+  if PRINT_DELTA_ONLY:
+    # Print the difference as a single series.
+    df_diff = df_sim['MIDPOINT'] - df_baseline['MIDPOINT']
+
+    # Smoothing.
+    df_diff = df_diff.rolling(window=10).mean()
+
+    df_diff.plot(color='C0', grid=True, linewidth=LW, ax=axes[0])
+
+    axes[0].legend(['Bid-ask Midpoint Delta'])
+  else:
+    df_baseline['MIDPOINT'].plot(color='C0', grid=True, linewidth=LW, ax=axes[0])
+    df_sim['MIDPOINT'].plot(color='C1', grid=True, linewidth=LW, alpha=0.9, ax=axes[0])
+
+    axes[0].legend(['Baseline', 'With Impact'])
+
+else:
+  #df_sim['PRICE'] = df_sim['PRICE'].rolling(window=sim_window).mean()
+
+  df_sim['MIDPOINT'].plot(color='C1', grid=True, linewidth=LW, alpha=0.9, ax=axes[0])
+  axes[0].legend(['Simulated'])
+
+
+plt.suptitle('Bid-Ask Midpoint: {}'.format(symbol))
+
+axes[0].set_ylabel('Quote Price')
+axes[0].set_xlabel('Quote Time')
+
+#plt.savefig('background_{}.png'.format(b))
+
+plt.show()
+