{value}
; - } + if ( probablyDate && - !isNaN(new Date(value).getTime()) && - !isoYearRegex.test(value?.toString()) + value?.length === 4 && + isoYearRegex.test(value?.toString()) ) { + return{value}
; + } + if (probablyDate && !isNaN(new Date(value).getTime())) { if (typeof value === "string") { const date = value.split("T")[0]; const time = value.split("T")[1]?.split(".")[0]; diff --git a/json_validate_exclude.txt b/json_validate_exclude.txt new file mode 100644 index 000000000000..da93f0e08ad6 --- /dev/null +++ b/json_validate_exclude.txt @@ -0,0 +1,3 @@ +tests/ +openbb_terminal/miscellaneous/gpt_index/index_0.4.json +.devcontainer/devcontainer.json \ No newline at end of file diff --git a/openbb_terminal/account/account_controller.py b/openbb_terminal/account/account_controller.py index 459675080ecd..8db4d421c25c 100644 --- a/openbb_terminal/account/account_controller.py +++ b/openbb_terminal/account/account_controller.py @@ -2,6 +2,7 @@ import logging from pathlib import Path from typing import Dict, List, Optional +from uuid import UUID from openbb_terminal.account.account_view import ( display_default_routines, @@ -55,12 +56,12 @@ def __init__(self, queue: Optional[List[str]] = None): """Constructor""" super().__init__(queue) self.LOCAL_ROUTINES: Dict[str, Path] = {} - self.REMOTE_CHOICES: List[str] = [] + self.REMOTE_CHOICES: Dict[str, UUID] = {} self.DEFAULT_ROUTINES: List[Dict[str, str]] = self.fetch_default_routines() - self.DEFAULT_CHOICES: List[str] = [ - r["name"] for r in self.DEFAULT_ROUTINES if "name" in r - ] + self.DEFAULT_CHOICES: Dict[str, None] = { + r["name"]: None for r in self.DEFAULT_ROUTINES if "name" in r + } if session and get_current_user().preferences.USE_PROMPT_TOOLKIT: self.choices: dict = self.choices_default @@ -75,7 +76,11 @@ def update_runtime_choices(self): {c: {} for c in self.LOCAL_ROUTINES} ) self.choices["download"]["--name"].update( - {c: {} for c in self.DEFAULT_CHOICES + self.REMOTE_CHOICES} + { + c: {} + for c in list(self.DEFAULT_CHOICES.keys()) + + list(self.REMOTE_CHOICES.keys()) + } ) self.choices["delete"]["--name"].update( {c: {} for c in self.REMOTE_CHOICES} @@ -250,10 +255,12 @@ def call_list(self, other_args: List[str]): auth_header=get_current_user().profile.get_auth_header(), page=ns_parser.page, size=ns_parser.size, + base_url=Hub.BackendEnvironment.BASE_URL, ) df, page, pages = get_personal_routines_info(response) if not df.empty: - self.REMOTE_CHOICES += list(df["name"]) + temp_dict = dict(zip(df["name"], df["uuid"])) + self.REMOTE_CHOICES = {**self.REMOTE_CHOICES, **temp_dict} self.update_runtime_choices() display_personal_routines(df, page, pages) else: @@ -350,6 +357,7 @@ def call_upload(self, other_args: List[str]): "routine": routine, "tags": tags, "public": ns_parser.public, + "base_url": Hub.BackendEnvironment.BASE_URL, } response = Hub.upload_routine(**kwargs) # type: ignore @@ -366,7 +374,8 @@ def call_upload(self, other_args: List[str]): console.print("[info]Aborted.[/info]") if response and response.status_code == 200: - self.REMOTE_CHOICES.append(name) + the_uuid = response.json()["uuid"] + self.REMOTE_CHOICES[name] = the_uuid self.update_runtime_choices() # store data in list with "personal/default" to identify data's routine type @@ -401,12 +410,14 @@ def call_download(self, other_args: List[str]): data = [] name = " ".join(ns_parser.name) # Personal routines - response = Hub.download_routine( - auth_header=get_current_user().profile.get_auth_header(), - name=name, - ) - if response and response.status_code == 200: - data = [response.json(), "personal"] + if name in self.REMOTE_CHOICES: + response = Hub.download_routine( + auth_header=get_current_user().profile.get_auth_header(), + uuid=self.REMOTE_CHOICES[name], + base_url=Hub.BackendEnvironment.BASE_URL, + ) + if response and response.status_code == 200: + data = [response.json(), "personal"] # Default routine elif name in self.DEFAULT_CHOICES: data = [ @@ -490,14 +501,15 @@ def call_delete(self, other_args: List[str]): if i.lower() in ["y", "yes"]: response = Hub.delete_routine( auth_header=get_current_user().profile.get_auth_header(), - name=name, + uuid=self.REMOTE_CHOICES[name], + base_url=Hub.BackendEnvironment.BASE_URL, ) if ( response and response.status_code == 200 and name in self.REMOTE_CHOICES ): - self.REMOTE_CHOICES.remove(name) + self.REMOTE_CHOICES.pop(name) self.update_runtime_choices() else: console.print("[info]Aborted.[/info]") @@ -543,6 +555,7 @@ def call_generate(self, other_args: List[str]) -> None: response = Hub.generate_personal_access_token( auth_header=get_current_user().profile.get_auth_header(), + base_url=Hub.BackendEnvironment.BASE_URL, days=ns_parser.days, ) if response and response.status_code == 200: diff --git a/openbb_terminal/account/account_view.py b/openbb_terminal/account/account_view.py index 45f2f93a4e2f..5dcbff376bfe 100644 --- a/openbb_terminal/account/account_view.py +++ b/openbb_terminal/account/account_view.py @@ -4,6 +4,34 @@ from openbb_terminal.rich_config import console +def clean_df(df: pd.DataFrame) -> pd.DataFrame: + """ + Cleans the dataframe before displaying it. + + Parameters + ---------- + df : pd.DataFrame + The dataframe to clean. + + Returns + ------- + pd.DataFrame + The cleaned dataframe. + """ + df["updated_date"] = pd.to_datetime(df["updated_date"]) + df["updated_date"] = df["updated_date"].dt.strftime("%Y-%m-%d %H:%M:%S") + df.replace(to_replace=[None], value="-", inplace=True) + to_rename = { + "name": "Name", + "description": "Description", + "version": "Version", + "updated_date": "Last Update", + } + df = df.rename(columns=to_rename) + df = df[["Name", "Description", "Version", "Last Update"]] + return df + + def display_personal_routines(df: pd.DataFrame, page: int, pages: int): """Display the routines. @@ -20,18 +48,17 @@ def display_personal_routines(df: pd.DataFrame, page: int, pages: int): title = f"Personal routines - page {page}" if pages: title += f" of {pages}" + df = clean_df(df) - df["updated_date"] = pd.to_datetime(df["updated_date"]) - df["updated_date"] = df["updated_date"].dt.strftime("%Y-%m-%d %H:%M:%S") - df.replace(to_replace=[None], value="-", inplace=True) print_rich_table( df=df, title=title, - headers=["Name", "Description", "Version", "Last update"], + headers=list(df.columns), show_index=True, index_name="#", ) - except Exception: + except Exception as exc: + print(exc) console.print("Failed to display personal routines.") @@ -44,9 +71,9 @@ def display_default_routines(df: pd.DataFrame): The default routines list. """ try: - df["date_updated"] = pd.to_datetime(df["date_updated"]) - df["date_updated"] = df["date_updated"].dt.strftime("%Y-%m-%d %H:%M:%S") - df.replace(to_replace=[None], value="-", inplace=True) + df = df.rename(columns={"date_updated": "updated_date"}) + print(df) + df = clean_df(df) print_rich_table( df=df, title="Default routines", diff --git a/openbb_terminal/common/behavioural_analysis/reddit_view.py b/openbb_terminal/common/behavioural_analysis/reddit_view.py index 8fc49af75065..0c0f87c46f9b 100644 --- a/openbb_terminal/common/behavioural_analysis/reddit_view.py +++ b/openbb_terminal/common/behavioural_analysis/reddit_view.py @@ -1,6 +1,7 @@ """Reddit View.""" __docformat__ = "numpy" +import io import logging import os import textwrap @@ -212,10 +213,10 @@ def display_wsb_community(limit: int = 10, new: bool = False): Flag to sort by new instead of hot, by default False """ subs = reddit_model.get_wsb_community(limit, new) + # I am not proud of this, but it works to eliminate the max recursion bug + subs = pd.read_csv(io.StringIO(subs.to_csv()), index_col=0).fillna("-") if not subs.empty: - # for sub in subs.iterrows(): - # print_reddit_post(sub) - print(print_rich_table(subs)) + print_rich_table(subs) @log_start_end(log=logger) @@ -282,8 +283,8 @@ def display_redditsent( Optionally specify the name of the sheet the data is exported to. export: str Format to export data - external_axes: Optional[List[plt.Axes]] - If supplied, expect 1 external axis + external_axes : bool, optional + Whether to return the figure object or not, by default False """ fig = OpenBBFigure() diff --git a/openbb_terminal/common/biztoc_model.py b/openbb_terminal/common/biztoc_model.py index 315e260a48f7..da3f120f79e5 100644 --- a/openbb_terminal/common/biztoc_model.py +++ b/openbb_terminal/common/biztoc_model.py @@ -46,6 +46,7 @@ def get_sources() -> pd.DataFrame: # If data request failed console.print("[red]Status code not 200. Unable to retrieve data\n[/red]") df = pd.DataFrame() + return df df = df.sort_values(by=["title"], ascending=True) df = df[["id", "title", "web"]] @@ -125,7 +126,7 @@ def get_news( "X-RapidAPI-Key": get_current_user().credentials.API_BIZTOC_TOKEN, "X-RapidAPI-Host": "biztoc.p.rapidapi.com", } - + df = pd.DataFrame() while not have_data: if term: req = request( @@ -158,7 +159,8 @@ def get_news( break df = pd.DataFrame(req.json(), columns=["title", "body", "url", "created"]) - + if df.empty: + return df df["created"] = pd.to_datetime(df["created"]) df = df.sort_values(by=[sort], ascending=False) diff --git a/openbb_terminal/common/quantitative_analysis/qa_view.py b/openbb_terminal/common/quantitative_analysis/qa_view.py index c273a12ba58c..83c26897eaba 100644 --- a/openbb_terminal/common/quantitative_analysis/qa_view.py +++ b/openbb_terminal/common/quantitative_analysis/qa_view.py @@ -121,7 +121,8 @@ def display_hist( fig.update_layout( xaxis_title="Value", - yaxis_title="Proportion", + margin=dict(r=40), + yaxis=dict(title="Proportion", title_standoff=40), bargap=0.01, bargroupgap=0, ) diff --git a/openbb_terminal/common/ultima_newsmonitor_model.py b/openbb_terminal/common/ultima_newsmonitor_model.py index b4e3782ac0da..656ba75c53fd 100644 --- a/openbb_terminal/common/ultima_newsmonitor_model.py +++ b/openbb_terminal/common/ultima_newsmonitor_model.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -BASE_URL = "https://api.ultimainsights.ai/v1" +ULTIMA_BASE_URL = "https://api.ultimainsights.ai/v1" NO_API_KEY = "REPLACE_ME" @@ -69,10 +69,10 @@ def get_news(term: str = "", sort: str = "articlePublishedDate") -> pd.DataFrame if term in supported_terms(): if auth_header: data = request( - f"{BASE_URL}/getNewsArticles/{term}", headers=auth_header + f"{ULTIMA_BASE_URL}/getNewsArticles/{term}", headers=auth_header ) else: - data = request(f"{BASE_URL}/getNewsArticles/{term}") + data = request(f"{ULTIMA_BASE_URL}/getNewsArticles/{term}") else: console.print( "[red]Ticker not supported. Unable to retrieve data\n[/red]" @@ -150,7 +150,7 @@ def supported_terms() -> list: # https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error/73270162#73270162 os.environ["REQUESTS_CA_BUNDLE"] = certifi.where() os.environ["SSL_CERT_FILE"] = certifi.where() - data = request(f"{BASE_URL}/supportedTickers") + data = request(f"{ULTIMA_BASE_URL}/supportedTickers") return list(data.json()) @@ -176,7 +176,7 @@ def get_company_info(ticker: str) -> dict: os.environ["SSL_CERT_FILE"] = certifi.where() if ticker in supported_terms(): - data = request(f"{BASE_URL}/getCompanyInfo/{ticker}") + data = request(f"{ULTIMA_BASE_URL}/getCompanyInfo/{ticker}") return data.json() console.print("[red]Ticker not supported. Unable to retrieve data\n[/red]") return {} @@ -213,9 +213,11 @@ def get_top_headlines(ticker: str) -> dict: if ticker in supported_terms(): if auth_header: - data = request(f"{BASE_URL}/getTopHeadlines/{ticker}", headers=auth_header) + data = request( + f"{ULTIMA_BASE_URL}/getTopHeadlines/{ticker}", headers=auth_header + ) else: - data = request(f"{BASE_URL}/getTopHeadlines/{ticker}") + data = request(f"{ULTIMA_BASE_URL}/getTopHeadlines/{ticker}") if ( hasattr(data, "status") and data.status_code == 429 ): # If data request failed diff --git a/openbb_terminal/core/completer/choices.py b/openbb_terminal/core/completer/choices.py index eff17a6f1fd0..fabfb46c384d 100644 --- a/openbb_terminal/core/completer/choices.py +++ b/openbb_terminal/core/completer/choices.py @@ -1,11 +1,11 @@ from argparse import ArgumentParser from contextlib import contextmanager from inspect import isfunction, unwrap -from os import environ from types import MethodType from typing import Callable, List from unittest.mock import patch +from openbb_terminal.core.session.current_system import get_current_system from openbb_terminal.helper_funcs import check_file_type_saved, check_positive from openbb_terminal.rich_config import get_ordered_list_sources @@ -211,7 +211,7 @@ def __patch_controller_functions(controller): ), ] - if str(environ.get("DEBUG_MODE", "false")).lower() != "true": + if not get_current_system().DEBUG_MODE: rich.start() patched_function_list = [] for patcher in patcher_list: @@ -219,7 +219,7 @@ def __patch_controller_functions(controller): yield patched_function_list - if str(environ.get("DEBUG_MODE", "false")).lower() != "true": + if not get_current_system().DEBUG_MODE: rich.stop() for patcher in patcher_list: patcher.stop() @@ -317,7 +317,7 @@ def build_controller_choice_map(controller) -> dict: argument_parser=argument_parser ) except Exception as exception: - if str(environ.get("DEBUG_MODE", "false")).lower() == "true": + if get_current_system().DEBUG_MODE: raise Exception( f"On command : `{command}`.\n{str(exception)}" ) from exception diff --git a/openbb_terminal/core/models/system_model.py b/openbb_terminal/core/models/system_model.py index 85d61e7b1bf0..7f478d304367 100644 --- a/openbb_terminal/core/models/system_model.py +++ b/openbb_terminal/core/models/system_model.py @@ -28,7 +28,7 @@ class SystemModel(BaseModel): PLATFORM: str = str(platform.platform()) # OpenBB section - VERSION: str = "3.2.0" + VERSION: str = "3.2.1" # Logging section LOGGING_APP_ID: str = "REPLACE_ME" @@ -54,6 +54,7 @@ class SystemModel(BaseModel): # Others TEST_MODE: bool = False DEBUG_MODE: bool = False + DEV_BACKEND: bool = False ENABLE_AUTHENTICATION: bool = True HEADLESS: bool = False diff --git a/openbb_terminal/core/plots/backend.py b/openbb_terminal/core/plots/backend.py index ba82753ab6d8..af6a29e29473 100644 --- a/openbb_terminal/core/plots/backend.py +++ b/openbb_terminal/core/plots/backend.py @@ -17,7 +17,9 @@ from packaging import version from reportlab.graphics import renderPDF -# pylint: disable=C0415 +from openbb_terminal.core.session.constants import BackendEnvironment + +# pylint: disable=C0411,C0412,C0415 try: from pywry import PyWry except ImportError as e: @@ -441,8 +443,10 @@ def call_hub(self, login: bool = True) -> Optional[dict]: self.check_backend() endpoint = {True: "login", False: "logout"}[login] + json_url = f"{BackendEnvironment.HUB_URL}{endpoint}?pywry=true" + outgoing = dict( - json_data=dict(url=f"https://my.openbb.co/{endpoint}?pywry=true"), + json_data=dict(url=json_url), **self.get_kwargs(endpoint.title()), width=900, height=800, diff --git a/openbb_terminal/core/plots/no_import.py b/openbb_terminal/core/plots/no_import.py index eb3fa1f10294..3d9a22d084cf 100644 --- a/openbb_terminal/core/plots/no_import.py +++ b/openbb_terminal/core/plots/no_import.py @@ -72,11 +72,11 @@ def __init__(self, daemon: bool = True, max_retries: int = 30): and get_current_system().LOGGING_SUB_APP != "sdk" ): console.print(pywry_missing) - if console.input( - "If you prefer to continue without interactive plots/tables, " - "press [green]enter[/] or [red]ctrl+c[/] to exit." - ): - dotenv.set_key(SETTINGS_ENV_FILE, "PLOT_ENABLE_PYWRY", "0") + console.print( + "\n\nContinuing with default plotly behaviour (open in browser).", + style="yellow", + ) + dotenv.set_key(SETTINGS_ENV_FILE, "PLOT_ENABLE_PYWRY", "0") current_user.preferences.USE_INTERACTIVE_DF = False set_current_user(current_user) diff --git a/openbb_terminal/core/plots/plotly_helper.py b/openbb_terminal/core/plots/plotly_helper.py index 68fb219d311f..dfa7f92aef0d 100644 --- a/openbb_terminal/core/plots/plotly_helper.py +++ b/openbb_terminal/core/plots/plotly_helper.py @@ -3,7 +3,7 @@ import json import sys import textwrap -from datetime import datetime +from datetime import datetime, timedelta from math import floor from pathlib import Path from typing import ( @@ -598,9 +598,9 @@ def _validate_x(data: Union[np.ndarray, pd.Series, type[TimeSeriesT]]): colors = [None] * len(valid_x) # type: ignore max_y = 0 - for i, (x_i, name_i, color_i) in enumerate(zip(valid_x, name, colors)): + for i0, (x_i, name_i, color_i) in enumerate(zip(valid_x, name, colors)): if not color_i: - color_i = theme.up_color if i % 2 == 0 else theme.down_color + color_i = theme.up_color if i0 % 2 == 0 else theme.down_color res_mean, res_std = np.mean(x_i), np.std(x_i) res_min, res_max = min(x_i), max(x_i) @@ -639,13 +639,13 @@ def _validate_x(data: Union[np.ndarray, pd.Series, type[TimeSeriesT]]): if show_rug: self.add_scatter( x=x_i, - y=[0.002] * len(x_i), + y=[0.00002] * len(x_i), name=name_i if len(name) < 2 else name[1], mode="markers", marker=dict( color=theme.down_color, symbol="line-ns-open", - size=8, + size=10, ), row=row, col=col, @@ -1340,6 +1340,7 @@ def hide_date_gaps( return # We get the missing days + is_daily = df_data.index[-1].time() == df_data.index[-2].time() dt_days = pd.date_range(start=dt_start, end=dt_end, normalize=True) # We get the dates that are missing @@ -1351,17 +1352,22 @@ def hide_date_gaps( if len(dt_missing_days) < 2_000: rangebreaks = [dict(values=dt_missing_days)] - # We get the frequency of the data to hide intra-day gaps - if df_data.index[-1].time() != df_data.index[-2].time(): - freq = df_data.index[1] - df_data.index[0] - freq_mins = int(freq.seconds / 60) - break_values = ( - df_data.resample(f"{freq_mins}T") - .max() - .index.union(df_data.index) - .difference(df_data.index) - ) - rangebreaks = [dict(values=break_values, dvalue=freq_mins * 60 * 1000)] + df_data = df_data.sort_index() + # We add a rangebreak if the first and second time are not the same + # since daily data will have the same time (00:00) + if not is_daily: + for i in range(len(df_data) - 1): + if df_data.index[i + 1] - df_data.index[i] > timedelta(hours=2): + rangebreaks.insert( + 0, + dict( + bounds=[ + df_data.index[i] + + timedelta(minutes=60 - df_data.index[i].minute), + df_data.index[i + 1], + ] + ), + ) self.update_xaxes(rangebreaks=rangebreaks, row=row, col=col) @@ -1372,16 +1378,19 @@ def add_rangebreaks(self) -> None: for row, row_dict in self._subplot_xdates.items(): for col, values in row_dict.items(): - x_values = ( - pd.to_datetime(np.concatenate(values)) - .to_pydatetime() - .astype("datetime64[ms]") - ) - self.hide_date_gaps( - pd.DataFrame(index=x_values.tolist()), - row=row, - col=col, - ) + try: + x_values = ( + pd.to_datetime(np.concatenate(values)) + .to_pydatetime() + .astype("datetime64[ms]") + ) + self.hide_date_gaps( + pd.DataFrame(index=x_values.tolist()), + row=row, + col=col, + ) + except ValueError: + continue def to_subplot( self, diff --git a/openbb_terminal/core/plots/table.html b/openbb_terminal/core/plots/table.html index 98161f928344..93a59444d995 100644 --- a/openbb_terminal/core/plots/table.html +++ b/openbb_terminal/core/plots/table.html @@ -79,7 +79,7 @@ }