Issue 183 (#185)

qf_lib/data_providers/bloomberg/bloomberg_data_provider.py

@@ -108,7 +108,8 @@ def connect(self):
         self.connected = True
 
     def _get_futures_chain_dict(self, tickers: Union[BloombergFutureTicker, Sequence[BloombergFutureTicker]],
-                                expiration_date_fields: Union[str, Sequence[str]]) -> Dict[BloombergFutureTicker, QFDataFrame]:
+                                expiration_date_fields: Union[str, Sequence[str]]) -> (
+            Dict)[BloombergFutureTicker, QFDataFrame]:
         """
         Returns tickers of futures contracts, which belong to the same futures contract chain as the provided ticker
         (tickers), along with their expiration dates.
@@ -145,7 +146,8 @@ def _get_futures_chain_dict(self, tickers: Union[BloombergFutureTicker, Sequence
             self._futures_data_provider.get_list_of_tickers_in_the_future_chain(tickers)
         all_specific_tickers = [ticker for specific_tickers_list in future_ticker_to_chain_tickers_list.values()
                                 for ticker in specific_tickers_list]
-        futures_expiration_dates = self.get_current_values(all_specific_tickers, expiration_date_fields).dropna(how="all")
+        futures_expiration_dates = self.get_current_values(all_specific_tickers, expiration_date_fields).dropna(
+            how="all")
 
         def specific_futures_index(future_ticker) -> pd.Index:
             """
@@ -291,12 +293,43 @@ def price_field_to_str_map(self) -> Dict[PriceField, str]:
         }
         return price_field_dict
 
-    def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None) -> List[BloombergTicker]:
+    def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None,
+                             display_figi: bool = False) -> List[BloombergTicker]:
+        """
+        Returns a list of all members of an index. It will not return any data for indices with more than
+        20,000 members.
+
+        Parameters
+        ----------
+        universe_ticker
+            ticker that describes a specific universe, which members will be returned
+        date
+            date for which current universe members' tickers will be returned
+        display_figi
+            the following flag can be used to have this field return Financial Instrument Global Identifiers (FIGI).
+        """
         date = date or datetime.now()
-        field = 'INDX_MWEIGHT_HIST'
-        ticker_data = self.get_tabular_data(universe_ticker, field, override_names="END_DT",
-                                            override_values=convert_to_bloomberg_date(date))
-        return [BloombergTicker(fields['Index Member'] + " Equity", SecurityType.STOCK, 1) for fields in ticker_data]
+        field = 'INDEX_MEMBERS_WEIGHTS'
+
+        MAX_PAGE_NUMBER = 7
+        MAX_MEMBERS_PER_PAGE = 3000
+        universe = []
+
+        def str_to_bbg_ticker(identifier: str, figi: bool):
+            ticker_str = f"/bbgid/{identifier}" if figi else f"{identifier} Equity"
+            return BloombergTicker(ticker_str, SecurityType.STOCK, 1)
+
+        for page_no in range(1, MAX_PAGE_NUMBER + 1):
+            ticker_data = self.get_tabular_data(universe_ticker, field,
+                                                ["END_DT", "PAGE_NUMBER_OVERRIDE", "DISPLAY_ID_BB_GLOBAL_OVERRIDE"],
+                                                [convert_to_bloomberg_date(date), page_no,
+                                                 "Y" if display_figi else "N"])
+            tickers_chunk = [str_to_bbg_ticker(fields['Index Member'], display_figi) for fields in ticker_data]
+            universe.extend(tickers_chunk)
+            if len(tickers_chunk) < MAX_MEMBERS_PER_PAGE:
+                break
+
+        return universe
 
     def get_unique_tickers(self, universe_ticker: Ticker) -> List[Ticker]:
         raise ValueError("BloombergDataProvider does not provide historical tickers_universe data")
@@ -333,7 +366,7 @@ def get_tabular_data(self, ticker: BloombergTicker, field: str,
         if override_names is not None:
             override_names, _ = convert_to_list(override_names, str)
         if override_values is not None:
-            override_values, _ = convert_to_list(override_values, str)
+            override_values, _ = convert_to_list(override_values, (str, int))
 
         tickers, got_single_ticker = convert_to_list(ticker, BloombergTicker)
         fields, got_single_field = convert_to_list(field, (PriceField, str))

qf_lib/data_providers/bloomberg/tabular_data_provider.py

@@ -14,7 +14,9 @@
 
 from typing import List
 
+from qf_lib.common.utils.logging.qf_parent_logger import qf_logger
 from qf_lib.data_providers.bloomberg.bloomberg_names import FIELD_DATA, REF_DATA_SERVICE_URI
+from qf_lib.data_providers.bloomberg.exceptions import BloombergError
 from qf_lib.data_providers.bloomberg.helpers import get_response_events, check_event_for_errors, extract_security_data, \
     check_security_data_for_errors, set_tickers, set_fields
 
@@ -30,6 +32,7 @@ class TabularDataProvider:
 
     def __init__(self, session):
         self._session = session
+        self.logger = qf_logger.getChild(self.__class__.__name__)
 
     def get(self, tickers, fields, override_names, override_values) -> List:
         ref_data_service = self._session.getService(REF_DATA_SERVICE_URI)
@@ -53,24 +56,27 @@ def _receive_reference_response(self, fields):
         elements = []
 
         for ev in response_events:
-            check_event_for_errors(ev)
-            security_data_array = extract_security_data(ev)
-            check_security_data_for_errors(security_data_array)
-
-            for i in range(security_data_array.numValues()):
-                security_data = security_data_array.getValueAsElement(i)
-                check_security_data_for_errors(security_data)
-
-                field_data_array = security_data.getElement(FIELD_DATA)
-
-                for field_name in fields:
-                    array = field_data_array.getElement(field_name)
-                    for element in array.values():
-                        keys_values_dict = {}
-                        for elem in element.elements():
-                            key = elem.name().__str__()
-                            value = element.getElementAsString(key)
-                            keys_values_dict[key] = value
-                        elements.append(keys_values_dict)
+            try:
+                check_event_for_errors(ev)
+                security_data_array = extract_security_data(ev)
+                check_security_data_for_errors(security_data_array)
+
+                for i in range(security_data_array.numValues()):
+                    security_data = security_data_array.getValueAsElement(i)
+                    check_security_data_for_errors(security_data)
+
+                    field_data_array = security_data.getElement(FIELD_DATA)
+
+                    for field_name in fields:
+                        array = field_data_array.getElement(field_name)
+                        for element in array.values():
+                            keys_values_dict = {}
+                            for elem in element.elements():
+                                key = elem.name().__str__()
+                                value = element.getElementAsString(key)
+                                keys_values_dict[key] = value
+                            elements.append(keys_values_dict)
+            except BloombergError as e:
+                self.logger.error(e)
 
         return elements

qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py

@@ -250,22 +250,46 @@ def expiration_date_field_str_map(self, ticker: BloombergTicker = None) -> Dict[
     def supported_ticker_types(self):
         return {BloombergTicker, BloombergFutureTicker}
 
-    def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None) -> List[BloombergTicker]:
+    def get_tickers_universe(self, universe_ticker: BloombergTicker, date: Optional[datetime] = None,
+                             display_figi: bool = False) -> List[BloombergTicker]:
         """
+        Returns a list of all members of an index. Bloomberg Data License supports only fetching constituents for
+        the current date and it will not return any data for indices with more than 20,000 members.
+
         Parameters
         ----------
         universe_ticker: BloombergTicker
             ticker that describes a specific universe, which members will be returned
         date: datetime
-            date for which current universe members' tickers will be returned
+            date for which current universe members' tickers will be returned.
+        display_figi
+            the following flag can be used to have this field return Financial Instrument Global Identifiers (FIGI).
         """
         date = date or datetime.now()
         if date.date() != datetime.today().date():
-            raise ValueError(f"{self.__class__.__name__} does not provide historical tickers_universe data")
+            raise ValueError(f"{self.__class__.__name__} does not provide historical tickers universe data")
+
+        field = 'INDEX_MEMBERS_WEIGHTS'
+
+        MAX_PAGE_NUMBER = 7
+        MAX_MEMBERS_PER_PAGE = 3000
+        universe = []
 
-        field = 'INDX_MEMBERS'
-        tickers: List[str] = self.get_current_values(universe_ticker, field)
-        return [BloombergTicker(f"{t} Equity", SecurityType.STOCK, 1) for t in tickers]
+        def str_to_bbg_ticker(data: str, figi: bool):
+            identifier = data.split(";")[0]
+            ticker_str = f"/bbgid/{identifier}" if figi else f"{identifier} Equity"
+            return BloombergTicker(ticker_str, SecurityType.STOCK, 1)
+
+        for page_no in range(1, MAX_PAGE_NUMBER + 1):
+            ticker_data = self.get_current_values(universe_ticker, field, fields_overrides=[
+                ("DISPLAY_ID_BB_GLOBAL_OVERRIDE", "Y" if display_figi else "N"),
+                ("PAGE_NUMBER_OVERRIDE", str(page_no))])
+            tickers_chunk = [str_to_bbg_ticker(data, display_figi) for data in ticker_data]
+            universe.extend(tickers_chunk)
+            if len(tickers_chunk) < MAX_MEMBERS_PER_PAGE:
+                break
+
+        return universe
 
     def get_unique_tickers(self, universe_ticker: BloombergTicker) -> List[BloombergTicker]:
         raise ValueError(f"{self.__class__.__name__} does not provide historical tickers_universe data")
@@ -379,7 +403,7 @@ def get_current_values(self, tickers: Union[BloombergTicker, Sequence[BloombergT
         fields, got_single_field = convert_to_list(fields, str)
 
         tickers_str_to_obj = {t.as_string(): t for t in tickers}
-        universe_id = self._get_universe_id(tickers, universe_creation_time)
+        universe_id = self._get_universe_id(tickers, universe_creation_time, fields_overrides)
         universe_url = self.universe_hapi_provider.get_universe_url(universe_id, list(tickers_str_to_obj.keys()),
                                                                     fields_overrides)
 
@@ -406,11 +430,12 @@ def get_current_values(self, tickers: Union[BloombergTicker, Sequence[BloombergT
         return cast_dataframe_to_proper_type(squeezed_result) if tickers_indices != 0 or fields_indices != 0 \
             else squeezed_result
 
-    def _get_universe_id(self, tickers: Sequence[BloombergTicker], creation_time: Optional[datetime] = None):
+    def _get_universe_id(self, tickers: Sequence[BloombergTicker], creation_time: Optional[datetime] = None,
+                         overrides: Optional[List[Tuple]] = None):
         universe_creation_time = creation_time or datetime.now()
         universe_id = f'uni{universe_creation_time:%m%d%H%M%S%f}'
 
-        if len(tickers) == 1:
+        if len(tickers) == 1 and not overrides:
             ticker_str = tickers[0].as_string().lower().replace(" ", "")
             universe_id = ticker_str if ticker_str.isalnum() else universe_id
 

qf_lib/tests/unit_tests/data_providers/bloomberg_beap_hapi/test_bloomberg_beap_hapi_data_provider.py

@@ -33,14 +33,14 @@ def test_get_tickers_universe__invalid_date(self):
 
     def test_get_tickers_universe__valid_ticker(self):
         self.data_provider.parser.get_current_values.return_value = QFDataFrame.from_records(
-            [(BloombergTicker("SPX Index"), ["Member1", "Member2"]), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker")
+            [(BloombergTicker("SPX Index"), ["Member1", "Member2"]), ], columns=["Ticker", "INDEX_MEMBERS_WEIGHTS"]).set_index("Ticker")
 
         universe = self.data_provider.get_tickers_universe(BloombergTicker("SPX Index"))
         self.assertCountEqual(universe, [BloombergTicker("Member1 Equity"), BloombergTicker("Member2 Equity")])
 
     def test_get_tickers_universe__invalid_ticker(self):
         self.data_provider.parser.get_current_values.return_value = QFDataFrame.from_records(
-            [(BloombergTicker("Invalid Index"), []), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker")
+            [(BloombergTicker("Invalid Index"), []), ], columns=["Ticker", "INDEX_MEMBERS_WEIGHTS"]).set_index("Ticker")
 
         universe = self.data_provider.get_tickers_universe(BloombergTicker("Invalid Index"))
         self.assertCountEqual(universe, [])