Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BugFix] Do Intrinio News TO-DOs -> Remove workaround because they fixed it. #6427

Merged
merged 3 commits into from
May 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions openbb_platform/openbb/assets/reference.json
Original file line number Diff line number Diff line change
Expand Up @@ -30534,15 +30534,15 @@
{
"name": "business_relevance_greater_than",
"type": "float",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
},
{
"name": "business_relevance_less_than",
"type": "float",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
Expand Down Expand Up @@ -31075,15 +31075,15 @@
{
"name": "business_relevance_greater_than",
"type": "float",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
},
{
"name": "business_relevance_less_than",
"type": "float",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source.",
"description": "News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1.",
"default": null,
"optional": true,
"choices": null
Expand Down
8 changes: 4 additions & 4 deletions openbb_platform/openbb/package/news.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@ def company(
is_spam : Optional[bool]
Filter whether it is marked as spam or not. Unsupported for yahoo source. (provider: intrinio)
business_relevance_greater_than : Optional[float]
News stories will have a business relevance score more than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
business_relevance_less_than : Optional[float]
News stories will have a business relevance score less than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
offset : Optional[int]
Page offset, used in conjunction with limit. (provider: tiingo)

Expand Down Expand Up @@ -327,9 +327,9 @@ def world(
is_spam : Optional[bool]
Filter whether it is marked as spam or not. Unsupported for yahoo source. (provider: intrinio)
business_relevance_greater_than : Optional[float]
News stories will have a business relevance score more than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score more than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
business_relevance_less_than : Optional[float]
News stories will have a business relevance score less than this value. Unsupported for yahoo source. (provider: intrinio)
News stories will have a business relevance score less than this value. Unsupported for yahoo source. Value is a decimal between 0 and 1. (provider: intrinio)
offset : Optional[int]
Page offset, used in conjunction with limit. (provider: tiingo)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
"""Intrinio Company News Model."""

import asyncio
from datetime import datetime
from datetime import datetime, timedelta
from typing import Any, Dict, List, Literal, Optional, Union
from warnings import warn

from openbb_core.provider.abstract.fetcher import Fetcher
from openbb_core.provider.standard_models.company_news import (
Expand Down Expand Up @@ -66,13 +67,17 @@ class IntrinioCompanyNewsQueryParams(CompanyNewsQueryParams):
)
business_relevance_greater_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score more than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)
business_relevance_less_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score less than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)


Expand Down Expand Up @@ -164,7 +169,18 @@ class IntrinioCompanyNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> IntrinioCompanyNewsQueryParams:
"""Transform the query params."""
return IntrinioCompanyNewsQueryParams(**params)
transformed_params = params
if not transformed_params.get("start_date"):
transformed_params["start_date"] = (
datetime.now() - timedelta(days=365)
).date()
if not transformed_params.get("end_date"):
transformed_params["end_date"] = (datetime.now() + timedelta(days=1)).date()
if transformed_params["start_date"] == transformed_params["end_date"]:
transformed_params["end_date"] = (
transformed_params["end_date"] + timedelta(days=1)
).date()
return IntrinioCompanyNewsQueryParams(**transformed_params)

@staticmethod
async def aextract_data(
Expand Down Expand Up @@ -199,23 +215,26 @@ async def callback(response, session):
# query.limit can be None...
limit = query.limit or 2500
while next_page and limit > articles:
url = f"{base_url}/{symbol}/news?{query_str}&api_key={api_key}&next_page={next_page}"
url = (
f"{base_url}/{symbol}/news?{query_str}"
+ f"&page_size={query.limit}&api_key={api_key}&next_page={next_page}"
)
result = await get_data(url, session=session, **kwargs)
_data = result.get("news", [])
if _data:
data.extend([{"symbol": symbol, **d} for d in _data])
articles = len(data)
next_page = result.get("next_page")
# Remove duplicates based on URL
return data

seen = set()

async def get_one(symbol):
"""Get the data for one symbol."""
# TODO: Change page_size to a more appropriate value when Intrinio fixes the bug in this param.
url = f"{base_url}/{symbol}/news?{query_str}&page_size=99&api_key={api_key}"
url = f"{base_url}/{symbol}/news?{query_str}&page_size={query.limit}&api_key={api_key}"
data = await amake_request(url, response_callback=callback, **kwargs)
if not data:
warn(f"No data found for: {symbol}")
if data:
data = [x for x in data if not (x["url"] in seen or seen.add(x["url"]))] # type: ignore
news.extend(
Expand All @@ -229,7 +248,10 @@ async def get_one(symbol):
await asyncio.gather(*tasks)

if not news:
raise EmptyDataError("Error: The request was returned as empty.")
raise EmptyDataError(
"Error: The request was returned as empty."
+ " Try adjusting the requested date ranges, if applicable."
)

return news

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Intrinio World News Model."""

from datetime import datetime
# pylint: disable=unused-argument

from datetime import datetime, timedelta
from typing import Any, Dict, List, Literal, Optional, Union

from openbb_core.provider.abstract.fetcher import Fetcher
Expand Down Expand Up @@ -60,13 +62,17 @@ class IntrinioWorldNewsQueryParams(WorldNewsQueryParams):
)
business_relevance_greater_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score more than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)
business_relevance_less_than: Optional[float] = Field(
default=None,
ge=0,
le=1,
description="News stories will have a business relevance score less than this value."
+ " Unsupported for yahoo source.",
+ " Unsupported for yahoo source. Value is a decimal between 0 and 1.",
)


Expand Down Expand Up @@ -161,7 +167,16 @@ class IntrinioWorldNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> IntrinioWorldNewsQueryParams:
"""Transform the query params."""
return IntrinioWorldNewsQueryParams(**params)
transformed_params = params
if not transformed_params.get("start_date"):
transformed_params["start_date"] = datetime.now().date()
if not transformed_params.get("end_date"):
transformed_params["end_date"] = (datetime.now() + timedelta(days=1)).date()
if transformed_params["start_date"] == transformed_params["end_date"]:
transformed_params["end_date"] = (
transformed_params["end_date"] + timedelta(days=1)
).date()
return IntrinioWorldNewsQueryParams(**transformed_params)

@staticmethod
async def aextract_data(
Expand All @@ -179,9 +194,7 @@ async def aextract_data(
else ["symbol", "page_size"]
)
query_str = get_querystring(query.model_dump(by_alias=True), ignore)
# TODO: Change page_size to a more appropriate value when Intrinio fixes the bug in this param.
url = f"{base_url}/news?{query_str}&page_size=99&api_key={api_key}"

url = f"{base_url}/news?{query_str}&page_size={query.limit}&api_key={api_key}"
seen = set()

async def callback(response, session):
Expand All @@ -195,7 +208,7 @@ async def callback(response, session):
articles = len(data)
next_page = result.get("next_page")
while next_page and articles < query.limit:
url = f"{base_url}/news?{query_str}&page_size=99&api_key={api_key}&next_page={next_page}"
url = f"{base_url}/news?{query_str}&page_size={query.limit}&api_key={api_key}&next_page={next_page}"
result = await get_data(url, session=session, **kwargs)
_data = result.get("news", [])
if _data:
Expand All @@ -215,14 +228,16 @@ async def callback(response, session):

return await amake_request(url, response_callback=callback, **kwargs) # type: ignore

# pylint: disable=unused-argument
@staticmethod
def transform_data(
query: IntrinioWorldNewsQueryParams, data: List[Dict], **kwargs: Any
) -> List[IntrinioWorldNewsData]:
"""Return the transformed data."""
if not data:
raise EmptyDataError("Error: The request was returned as empty.")
raise EmptyDataError(
"Error: The request was returned as empty."
+ " Try adjusting the requested date ranges, if applicable."
)
results: List[IntrinioWorldNewsData] = []
for item in data:
body = item.get("body", {})
Expand Down
Loading
Loading