diff --git a/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py b/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py index 4ff83f7e..4ddcdcf3 100644 --- a/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py +++ b/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py @@ -108,7 +108,8 @@ def connect(self): self.connected = True def _get_futures_chain_dict(self, tickers: Union[BloombergFutureTicker, Sequence[BloombergFutureTicker]], - expiration_date_fields: Union[str, Sequence[str]]) -> Dict[BloombergFutureTicker, QFDataFrame]: + expiration_date_fields: Union[str, Sequence[str]]) -> ( + Dict)[BloombergFutureTicker, QFDataFrame]: """ Returns tickers of futures contracts, which belong to the same futures contract chain as the provided ticker (tickers), along with their expiration dates. @@ -145,7 +146,8 @@ def _get_futures_chain_dict(self, tickers: Union[BloombergFutureTicker, Sequence self._futures_data_provider.get_list_of_tickers_in_the_future_chain(tickers) all_specific_tickers = [ticker for specific_tickers_list in future_ticker_to_chain_tickers_list.values() for ticker in specific_tickers_list] - futures_expiration_dates = self.get_current_values(all_specific_tickers, expiration_date_fields).dropna(how="all") + futures_expiration_dates = self.get_current_values(all_specific_tickers, expiration_date_fields).dropna( + how="all") def specific_futures_index(future_ticker) -> pd.Index: """ @@ -291,12 +293,43 @@ def price_field_to_str_map(self) -> Dict[PriceField, str]: } return price_field_dict - def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None) -> List[BloombergTicker]: + def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None, + display_figi: bool = False) -> List[BloombergTicker]: + """ + Returns a list of all members of an index. It will not return any data for indices with more than + 20,000 members. + + Parameters + ---------- + universe_ticker + ticker that describes a specific universe, which members will be returned + date + date for which current universe members' tickers will be returned + display_figi + the following flag can be used to have this field return Financial Instrument Global Identifiers (FIGI). + """ date = date or datetime.now() - field = 'INDX_MWEIGHT_HIST' - ticker_data = self.get_tabular_data(universe_ticker, field, override_names="END_DT", - override_values=convert_to_bloomberg_date(date)) - return [BloombergTicker(fields['Index Member'] + " Equity", SecurityType.STOCK, 1) for fields in ticker_data] + field = 'INDEX_MEMBERS_WEIGHTS' + + MAX_PAGE_NUMBER = 7 + MAX_MEMBERS_PER_PAGE = 3000 + universe = [] + + def str_to_bbg_ticker(identifier: str, figi: bool): + ticker_str = f"/bbgid/{identifier}" if figi else f"{identifier} Equity" + return BloombergTicker(ticker_str, SecurityType.STOCK, 1) + + for page_no in range(1, MAX_PAGE_NUMBER + 1): + ticker_data = self.get_tabular_data(universe_ticker, field, + ["END_DT", "PAGE_NUMBER_OVERRIDE", "DISPLAY_ID_BB_GLOBAL_OVERRIDE"], + [convert_to_bloomberg_date(date), page_no, + "Y" if display_figi else "N"]) + tickers_chunk = [str_to_bbg_ticker(fields['Index Member'], display_figi) for fields in ticker_data] + universe.extend(tickers_chunk) + if len(tickers_chunk) < MAX_MEMBERS_PER_PAGE: + break + + return universe def get_unique_tickers(self, universe_ticker: Ticker) -> List[Ticker]: raise ValueError("BloombergDataProvider does not provide historical tickers_universe data") @@ -333,7 +366,7 @@ def get_tabular_data(self, ticker: BloombergTicker, field: str, if override_names is not None: override_names, _ = convert_to_list(override_names, str) if override_values is not None: - override_values, _ = convert_to_list(override_values, str) + override_values, _ = convert_to_list(override_values, (str, int)) tickers, got_single_ticker = convert_to_list(ticker, BloombergTicker) fields, got_single_field = convert_to_list(field, (PriceField, str)) diff --git a/qf_lib/data_providers/bloomberg/tabular_data_provider.py b/qf_lib/data_providers/bloomberg/tabular_data_provider.py index 6bd9cfdb..8b4bfc12 100644 --- a/qf_lib/data_providers/bloomberg/tabular_data_provider.py +++ b/qf_lib/data_providers/bloomberg/tabular_data_provider.py @@ -14,7 +14,9 @@ from typing import List +from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.data_providers.bloomberg.bloomberg_names import FIELD_DATA, REF_DATA_SERVICE_URI +from qf_lib.data_providers.bloomberg.exceptions import BloombergError from qf_lib.data_providers.bloomberg.helpers import get_response_events, check_event_for_errors, extract_security_data, \ check_security_data_for_errors, set_tickers, set_fields @@ -30,6 +32,7 @@ class TabularDataProvider: def __init__(self, session): self._session = session + self.logger = qf_logger.getChild(self.__class__.__name__) def get(self, tickers, fields, override_names, override_values) -> List: ref_data_service = self._session.getService(REF_DATA_SERVICE_URI) @@ -53,24 +56,27 @@ def _receive_reference_response(self, fields): elements = [] for ev in response_events: - check_event_for_errors(ev) - security_data_array = extract_security_data(ev) - check_security_data_for_errors(security_data_array) - - for i in range(security_data_array.numValues()): - security_data = security_data_array.getValueAsElement(i) - check_security_data_for_errors(security_data) - - field_data_array = security_data.getElement(FIELD_DATA) - - for field_name in fields: - array = field_data_array.getElement(field_name) - for element in array.values(): - keys_values_dict = {} - for elem in element.elements(): - key = elem.name().__str__() - value = element.getElementAsString(key) - keys_values_dict[key] = value - elements.append(keys_values_dict) + try: + check_event_for_errors(ev) + security_data_array = extract_security_data(ev) + check_security_data_for_errors(security_data_array) + + for i in range(security_data_array.numValues()): + security_data = security_data_array.getValueAsElement(i) + check_security_data_for_errors(security_data) + + field_data_array = security_data.getElement(FIELD_DATA) + + for field_name in fields: + array = field_data_array.getElement(field_name) + for element in array.values(): + keys_values_dict = {} + for elem in element.elements(): + key = elem.name().__str__() + value = element.getElementAsString(key) + keys_values_dict[key] = value + elements.append(keys_values_dict) + except BloombergError as e: + self.logger.error(e) return elements diff --git a/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py b/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py index 3b74b40f..b762b57a 100644 --- a/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py +++ b/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py @@ -250,22 +250,46 @@ def expiration_date_field_str_map(self, ticker: BloombergTicker = None) -> Dict[ def supported_ticker_types(self): return {BloombergTicker, BloombergFutureTicker} - def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None) -> List[BloombergTicker]: + def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None, + display_figi: bool = False) -> List[BloombergTicker]: """ + Returns a list of all members of an index. Bloomberg Data License supports only fetching constituents for + the current date and it will not return any data for indices with more than 20,000 members. + Parameters ---------- universe_ticker: BloombergTicker ticker that describes a specific universe, which members will be returned date: datetime - date for which current universe members' tickers will be returned + date for which current universe members' tickers will be returned. + display_figi + the following flag can be used to have this field return Financial Instrument Global Identifiers (FIGI). """ date = date or datetime.now() if date.date() != datetime.today().date(): - raise ValueError(f"{self.__class__.__name__} does not provide historical tickers_universe data") + raise ValueError(f"{self.__class__.__name__} does not provide historical tickers universe data") + + field = 'INDEX_MEMBERS_WEIGHTS' + + MAX_PAGE_NUMBER = 7 + MAX_MEMBERS_PER_PAGE = 3000 + universe = [] - field = 'INDX_MEMBERS' - tickers: List[str] = self.get_current_values(universe_ticker, field) - return [BloombergTicker(f"{t} Equity", SecurityType.STOCK, 1) for t in tickers] + def str_to_bbg_ticker(data: str, figi: bool): + identifier = data.split(";")[0] + ticker_str = f"/bbgid/{identifier}" if figi else f"{identifier} Equity" + return BloombergTicker(ticker_str, SecurityType.STOCK, 1) + + for page_no in range(1, MAX_PAGE_NUMBER + 1): + ticker_data = self.get_current_values(universe_ticker, field, fields_overrides=[ + ("DISPLAY_ID_BB_GLOBAL_OVERRIDE", "Y" if display_figi else "N"), + ("PAGE_NUMBER_OVERRIDE", str(page_no))]) + tickers_chunk = [str_to_bbg_ticker(data, display_figi) for data in ticker_data] + universe.extend(tickers_chunk) + if len(tickers_chunk) < MAX_MEMBERS_PER_PAGE: + break + + return universe def get_unique_tickers(self, universe_ticker: BloombergTicker) -> List[BloombergTicker]: raise ValueError(f"{self.__class__.__name__} does not provide historical tickers_universe data") @@ -379,7 +403,7 @@ def get_current_values(self, tickers: Union[BloombergTicker, Sequence[BloombergT fields, got_single_field = convert_to_list(fields, str) tickers_str_to_obj = {t.as_string(): t for t in tickers} - universe_id = self._get_universe_id(tickers, universe_creation_time) + universe_id = self._get_universe_id(tickers, universe_creation_time, fields_overrides) universe_url = self.universe_hapi_provider.get_universe_url(universe_id, list(tickers_str_to_obj.keys()), fields_overrides) @@ -406,11 +430,12 @@ def get_current_values(self, tickers: Union[BloombergTicker, Sequence[BloombergT return cast_dataframe_to_proper_type(squeezed_result) if tickers_indices != 0 or fields_indices != 0 \ else squeezed_result - def _get_universe_id(self, tickers: Sequence[BloombergTicker], creation_time: Optional[datetime] = None): + def _get_universe_id(self, tickers: Sequence[BloombergTicker], creation_time: Optional[datetime] = None, + overrides: Optional[List[Tuple]] = None): universe_creation_time = creation_time or datetime.now() universe_id = f'uni{universe_creation_time:%m%d%H%M%S%f}' - if len(tickers) == 1: + if len(tickers) == 1 and not overrides: ticker_str = tickers[0].as_string().lower().replace(" ", "") universe_id = ticker_str if ticker_str.isalnum() else universe_id diff --git a/qf_lib/tests/unit_tests/data_providers/bloomberg_beap_hapi/test_bloomberg_beap_hapi_data_provider.py b/qf_lib/tests/unit_tests/data_providers/bloomberg_beap_hapi/test_bloomberg_beap_hapi_data_provider.py index 4f94f845..2c66211f 100644 --- a/qf_lib/tests/unit_tests/data_providers/bloomberg_beap_hapi/test_bloomberg_beap_hapi_data_provider.py +++ b/qf_lib/tests/unit_tests/data_providers/bloomberg_beap_hapi/test_bloomberg_beap_hapi_data_provider.py @@ -33,14 +33,14 @@ def test_get_tickers_universe__invalid_date(self): def test_get_tickers_universe__valid_ticker(self): self.data_provider.parser.get_current_values.return_value = QFDataFrame.from_records( - [(BloombergTicker("SPX Index"), ["Member1", "Member2"]), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker") + [(BloombergTicker("SPX Index"), ["Member1", "Member2"]), ], columns=["Ticker", "INDEX_MEMBERS_WEIGHTS"]).set_index("Ticker") universe = self.data_provider.get_tickers_universe(BloombergTicker("SPX Index")) self.assertCountEqual(universe, [BloombergTicker("Member1 Equity"), BloombergTicker("Member2 Equity")]) def test_get_tickers_universe__invalid_ticker(self): self.data_provider.parser.get_current_values.return_value = QFDataFrame.from_records( - [(BloombergTicker("Invalid Index"), []), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker") + [(BloombergTicker("Invalid Index"), []), ], columns=["Ticker", "INDEX_MEMBERS_WEIGHTS"]).set_index("Ticker") universe = self.data_provider.get_tickers_universe(BloombergTicker("Invalid Index")) self.assertCountEqual(universe, [])