From 84396543d684734d3aac6ea897fd14aa935f568d Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 1 May 2024 18:36:45 -0700 Subject: [PATCH 1/3] try catch for RemoteDisconnect error --- .../sec/openbb_sec/models/etf_holdings.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py b/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py index 15695e7f02b1..e5ac44c69432 100644 --- a/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py +++ b/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py @@ -2,6 +2,7 @@ # pylint: disable =[unused-argument,too-many-locals,too-many-branches] +import asyncio from datetime import date as dateType from typing import Any, Dict, List, Optional, Union from warnings import warn @@ -329,9 +330,22 @@ async def aextract_data( **kwargs: Any, ) -> Dict: """Return the raw data from the SEC endpoint.""" - filings = await get_nport_candidates( - symbol=query.symbol, use_cache=query.use_cache - ) + # Implement a retry mechanism in case of RemoteDiconnected Error. + retries = 3 + for i in range(retries): + filings = [] + try: + filings = await get_nport_candidates( + symbol=query.symbol, use_cache=query.use_cache + ) + if filings: + break + except Exception as e: + if i < retries - 1: + warn(f"Error: {e}. Retrying...") + asyncio.sleep(1) + continue + raise e filing_candidates = pd.DataFrame.from_records(filings) if filing_candidates.empty: raise ValueError(f"No N-Port records found for {query.symbol}.") From 51c80ff67bd4430312db419e523e1463792d5161 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 1 May 2024 18:52:31 -0700 Subject: [PATCH 2/3] await sleep --- openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py b/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py index e5ac44c69432..89b65b42d64a 100644 --- a/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py +++ b/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py @@ -343,7 +343,7 @@ async def aextract_data( except Exception as e: if i < retries - 1: warn(f"Error: {e}. Retrying...") - asyncio.sleep(1) + await asyncio.sleep(1) continue raise e filing_candidates = pd.DataFrame.from_records(filings) From 152955546a995f4ae548d708f922f4c9c3edef8f Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 1 May 2024 22:22:12 -0700 Subject: [PATCH 3/3] remove not implemented helper --- .../providers/sec/openbb_sec/utils/helpers.py | 88 +------------------ 1 file changed, 1 insertion(+), 87 deletions(-) diff --git a/openbb_platform/providers/sec/openbb_sec/utils/helpers.py b/openbb_platform/providers/sec/openbb_sec/utils/helpers.py index 0f72bcdd5ed1..407a8fc773b4 100644 --- a/openbb_platform/providers/sec/openbb_sec/utils/helpers.py +++ b/openbb_platform/providers/sec/openbb_sec/utils/helpers.py @@ -2,19 +2,16 @@ # pylint: disable =unused-argument -from datetime import timedelta from io import BytesIO from typing import Dict, List, Optional, Union from zipfile import ZipFile import pandas as pd -import requests -import requests_cache from aiohttp_client_cache import SQLiteBackend from aiohttp_client_cache.session import CachedSession from openbb_core.app.utils import get_user_cache_directory from openbb_core.provider.utils.helpers import amake_request, make_request -from openbb_sec.utils.definitions import HEADERS, QUARTERS, SEC_HEADERS, TAXONOMIES +from openbb_sec.utils.definitions import HEADERS, SEC_HEADERS async def sec_callback(response, session): @@ -167,89 +164,6 @@ async def cik_map(cik: Union[str, int], use_cache: bool = True) -> str: return symbol -def get_frame( # pylint: disable =too-many-arguments - year: int, - quarter: Optional[QUARTERS] = None, - taxonomy: TAXONOMIES = "us-gaap", - units: str = "USD", - fact: str = "Revenues", - instantaneous: bool = False, - use_cache: bool = True, -) -> Dict: - """Get a frame of data for a given fact. - - The xbrl/frames API aggregates one fact for each reporting entity - that is last filed that most closely fits the calendrical period requested. - - This API supports for annual, quarterly and instantaneous data: - - https://data.sec.gov/api/xbrl/frames/us-gaap/AccountsPayableCurrent/USD/CY2019Q1I.json - - Where the units of measure specified in the XBRL contains a numerator and a denominator, - these are separated by “-per-” such as “USD-per-shares”. Note that the default unit in XBRL is “pure”. - - CY####Q# for quarterly data (duration 91 days +/- 30 days). - Because company financial calendars can start and end on any month or day and even change in length from quarter to - quarter according to the day of the week, the frame data is assembled by the dates that best align with a calendar - quarter or year. Data users should be mindful different reporting start and end dates for facts contained in a frame. - - Example facts: - Revenues - GrossProfit - CostOfRevenue - DividendsCash - DistributedEarnings - AccountsPayableCurrent - OperatingExpenses - OperatingIncomeLoss - NoninterestIncome - InterestAndDebtExpense - IncomeTaxExpenseBenefit - NetIncomeLoss - - Facts where units are, "shares": - WeightedAverageNumberOfDilutedSharesOutstanding - """ - if fact in ["WeightedAverageNumberOfDilutedSharesOutstanding"]: - units = "shares" - sec_session_frames = requests_cache.CachedSession( - f"{get_user_cache_directory()}/http/sec_frames", expire_after=timedelta(days=2) - ) - url = f"https://data.sec.gov/api/xbrl/frames/{taxonomy}/{fact}/{units}/CY{year}" - - if quarter: - url = url + f"Q{quarter}" - - if instantaneous: - url = url + "I" - url = url + ".json" - r = ( - requests.get(url, headers=HEADERS, timeout=5) - if use_cache is False - else sec_session_frames.get(url, headers=HEADERS, timeout=5) - ) - - if r.status_code != 200: - raise RuntimeError(f"Request failed with status code {r.status_code}") - - response = r.json() - - data = sorted(response["data"], key=lambda x: x["val"], reverse=True) - metadata = { - "frame": response["ccp"], - "tag": response["tag"], - "label": response["label"], - "description": response["description"], - "taxonomy": response["taxonomy"], - "unit": response["uom"], - "count": response["pts"], - } - - results = {"metadata": metadata, "data": data} - - return results - - def get_schema_filelist(query: str = "", url: str = "", use_cache: bool = True) -> List: """Get a list of schema files from the SEC website.""" results: List = []