diff --git a/Algorithm.CSharp/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.cs b/Algorithm.CSharp/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.cs
index c379ed4f6ffa..4a424dccfba8 100644
--- a/Algorithm.CSharp/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.cs
+++ b/Algorithm.CSharp/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.cs
@@ -22,8 +22,8 @@
namespace QuantConnect.Algorithm.CSharp
{
///
- /// Regression algorithm that asserts Stochastic indicator, registered with a different resolution consolidator,
- /// is warmed up properly by calling QCAlgorithm.WarmUpIndicator
+ /// This regression algorithm asserts the consolidated US equity daily bars from the hour bars exactly matches
+ /// the daily bars returned from the database
///
public class ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
{
@@ -39,9 +39,17 @@ public override void Initialize()
SetEndDate(2020, 6, 5);
_spy = AddEquity("SPY", Resolution.Hour).Symbol;
+
+ // We will use these two indicators to compare the daily consolidated bars equals
+ // the ones returned from the database. We use this specific type of indicator as
+ // it depends on its previous values. Thus, if at some point the bars received by
+ // the indicators differ, so will their final values
_rsi = new RelativeStrengthIndex("FIRST", 15, MovingAverageType.Wilders);
RegisterIndicator(_spy, _rsi, Resolution.Daily);
+ // We won't register this indicator as we will update it manually at the end of the
+ // month, so that we can compare the values of the indicator that received consolidated
+ // bars and the values of this one
_rsiTimeDelta = new RelativeStrengthIndex("SECOND" ,15, MovingAverageType.Wilders);
}
@@ -68,6 +76,10 @@ public override void OnData(Slice slice)
else
{
_values[Time.Date] = _rsi.Current.Value;
+
+ // Since the symbol resolution is hour and the symbol is equity, we know the last bar received in a day will
+ // be at the market close, this is 16h. We need to count how many daily bars were consolidated in order to know
+ // how many we need to request from the history
if (Time.Hour == 16)
{
_count++;
diff --git a/Algorithm.Python/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.py b/Algorithm.Python/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.py
index e694d6205049..45d3c5f3c831 100644
--- a/Algorithm.Python/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.py
+++ b/Algorithm.Python/ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm.py
@@ -13,20 +13,28 @@
from AlgorithmImports import *
+###
+### This regression algorithm asserts the consolidated US equity daily bars from the hour bars exactly matches
+### the daily bars returned from the database
+###
class ConsolidateHourBarsIntoDailyBarsRegressionAlgorithm(QCAlgorithm):
def initialize(self):
# change the start date between runs to check that warm up shows the correct value
self.set_start_date(2020, 5, 1)
self.set_end_date(2020, 6, 5)
- self.set_cash(100000)
self.spy = self.add_equity("SPY", Resolution.HOUR).symbol
- # Resolution.DAILY indicators
+ # We will use these two indicators to compare the daily consolidated bars equals
+ # the ones returned from the database. We use this specific type of indicator as
+ # it depends on its previous values. Thus, if at some point the bars received by
+ # the indicators differ, so will their final values
self._rsi = RelativeStrengthIndex("First", 15, MovingAverageType.WILDERS)
self.register_indicator(self.spy, self._rsi, Resolution.DAILY)
- # Resolution.DAILY indicators
+ # We won't register this indicator as we will update it manually at the end of the
+ # month, so that we can compare the values of the indicator that received consolidated
+ # bars and the values of this one
self._rsi_timedelta = RelativeStrengthIndex("Second", 15, MovingAverageType.WILDERS)
self._values = {}
self.count = 0;
@@ -47,5 +55,9 @@ def on_data(self, data: Slice):
else:
time = self.time.strftime('%Y-%m-%d')
self._values[time] = self._rsi.current.value
+
+ # Since the symbol resolution is hour and the symbol is equity, we know the last bar received in a day will
+ # be at the market close, this is 16h. We need to count how many daily bars were consolidated in order to know
+ # how many we need to request from the history
if self.time.hour == 16:
self.count += 1
diff --git a/Common/Data/Consolidators/MarketHourAwareConsolidator.cs b/Common/Data/Consolidators/MarketHourAwareConsolidator.cs
index f0e7fd7de2ce..2b9565f5cb24 100644
--- a/Common/Data/Consolidators/MarketHourAwareConsolidator.cs
+++ b/Common/Data/Consolidators/MarketHourAwareConsolidator.cs
@@ -132,9 +132,12 @@ public virtual void Update(IBaseData data)
{
Initialize(data);
+ // US equity hour data from the database starts at 9am but the exchange opens at 9:30am. Thus, we need to handle
+ // this case specifically to avoid skipping the first hourly bar. To avoid this, we assert the period is daily,
+ // the data resolution is hour and the exchange opens at any point in time over the data.Time to data.EndTime interval
if (_extendedMarketHours ||
ExchangeHours.IsOpen(data.Time, false) ||
- (Period == TimeSpan.FromDays(1) && ExchangeHours.IsOpen(data.Time, data.EndTime, false)))
+ (Period == Time.OneDay && (data.EndTime - data.Time == Time.OneHour) && ExchangeHours.IsOpen(data.Time, data.EndTime, false)))
{
Consolidator.Update(data);
}
diff --git a/Common/Data/Consolidators/PeriodCountConsolidatorBase.cs b/Common/Data/Consolidators/PeriodCountConsolidatorBase.cs
index ead3542869a8..7e76d9050bc3 100644
--- a/Common/Data/Consolidators/PeriodCountConsolidatorBase.cs
+++ b/Common/Data/Consolidators/PeriodCountConsolidatorBase.cs
@@ -322,13 +322,17 @@ protected DateTime GetRoundedBarTime(DateTime time)
protected DateTime GetRoundedBarTime(IBaseData inputData)
{
var potentialStartTime = GetRoundedBarTime(inputData.Time);
- if (_period.HasValue)
+ if (_period.HasValue && potentialStartTime + _period < inputData.EndTime)
{
+ // US equity hour bars from the database start at 9am but the exchange opens at 9:30am. Thus, the method
+ // GetRoundedBarTime(inputData.Time) returns the market open of the previous day, which is not consistent
+ // with the given end time. However, while the date returned is incorrect, the time of day isn't. For that
+ // reason we need to handle this case specifically.
if (inputData.EndTime - inputData.Time == TimeSpan.FromHours(1) && potentialStartTime.Date < inputData.Time.Date)
{
potentialStartTime = inputData.Time.Date + potentialStartTime.TimeOfDay;
}
- else if (potentialStartTime + _period < inputData.EndTime)
+ else
{
// whops! the end time we were giving is beyond our potential end time, so let's use the giving bars star time instead
potentialStartTime = inputData.Time;
diff --git a/Tests/Common/Data/MarketHourAwareConsolidatorTests.cs b/Tests/Common/Data/MarketHourAwareConsolidatorTests.cs
index 14dc97c84367..d577c73000bd 100644
--- a/Tests/Common/Data/MarketHourAwareConsolidatorTests.cs
+++ b/Tests/Common/Data/MarketHourAwareConsolidatorTests.cs
@@ -117,6 +117,54 @@ public void Daily(bool strictEndTime)
Assert.AreEqual(1, latestBar.Low);
}
+ [Test]
+ public void FirstHourBarIsNotSkippedWhenBarResolutionIsHour()
+ {
+ var symbol = Symbols.SPY;
+ using var consolidator = new MarketHourAwareConsolidator(true, Resolution.Daily, typeof(TradeBar), TickType.Trade, false);
+ var consolidatedBarsCount = 0;
+ TradeBar latestBar = null;
+
+ consolidator.DataConsolidated += (sender, bar) =>
+ {
+ latestBar = (TradeBar)bar;
+ consolidatedBarsCount++;
+ };
+
+ var time = new DateTime(2020, 05, 01, 09, 30, 0);
+ // this bar will be ignored because it's during market closed hours and the bar resolution is not Hour
+ consolidator.Update(new TradeBar() { Time = time.Subtract(Time.OneMinute), Period = Time.OneMinute, Symbol = symbol, Open = 1});
+
+ time = new DateTime(2020, 05, 01, 09, 0, 0);
+ var hourBars = new List()
+ {
+ new TradeBar() { Time = time, Period = Time.OneHour, Symbol = symbol, Open = 2 },
+ new TradeBar() { Time = time.AddHours(1), Period = Time.OneHour, Symbol = symbol, High = 200 },
+ new TradeBar() { Time = time.AddHours(2), Period = Time.OneHour, Symbol = symbol, Low = 0.02m },
+ new TradeBar() { Time = time.AddHours(3), Period = Time.OneHour, Symbol = symbol, Close = 20 },
+ new TradeBar() { Time = time.AddHours(4), Period = Time.OneHour, Symbol = symbol, Open = 3 },
+ new TradeBar() { Time = time.AddHours(5), Period = Time.OneHour, Symbol = symbol, High = 300 },
+ new TradeBar() { Time = time.AddHours(6), Period = Time.OneHour, Symbol = symbol, Low = 0.03m, Close = 30 },
+ };
+
+ foreach (var bar in hourBars)
+ {
+ consolidator.Update(bar);
+ }
+
+ consolidator.Scan(time.AddHours(7));
+
+ // Assert that the bar emitted
+ Assert.IsNotNull(latestBar);
+ Assert.AreEqual(time.AddHours(7), latestBar.EndTime);
+ Assert.AreEqual(time.AddMinutes(30), latestBar.Time);
+ Assert.AreEqual(1, consolidatedBarsCount);
+ Assert.AreEqual(2, latestBar.Open);
+ Assert.AreEqual(300, latestBar.High);
+ Assert.AreEqual(0.02, latestBar.Low);
+ Assert.AreEqual(30, latestBar.Close);
+ }
+
[TestCase(true)]
[TestCase(false)]
public void DailyExtendedMarketHours(bool strictEndTime)