diff --git a/openbb_terminal/terminal_controller.py b/openbb_terminal/terminal_controller.py
index d532fb10c04f..50133a565999 100644
--- a/openbb_terminal/terminal_controller.py
+++ b/openbb_terminal/terminal_controller.py
@@ -21,7 +21,6 @@
from prompt_toolkit.completion import NestedCompleter
from prompt_toolkit.formatted_text import HTML
from prompt_toolkit.styles import Style
-from rich import panel
import openbb_terminal.config_terminal as cfg
from openbb_terminal.account.account_controller import (
@@ -501,16 +500,17 @@ def call_funds(self, _):
def call_intro(self, _):
"""Process intro command."""
- import json
+ webbrowser.open("https://docs.openbb.co/terminal/usage/basics")
+ # import json
- intro: dict = json.load((Path(__file__).parent / "intro.json").open()) # type: ignore
+ # intro: dict = json.load((Path(__file__).parent / "intro.json").open()) # type: ignore
- for prompt in intro.get("prompts", []):
- console.print(panel.Panel(f"[purple]{prompt['header']}[/purple]"))
- console.print("".join(prompt["content"]))
- if input("") == "q":
- break
- console.print("\n")
+ # for prompt in intro.get("prompts", []):
+ # console.print(panel.Panel(f"[purple]{prompt['header']}[/purple]"))
+ # console.print("".join(prompt["content"]))
+ # if input("") == "q":
+ # break
+ # console.print("\n")
def call_exe(self, other_args: List[str]):
"""Process exe command."""
@@ -705,11 +705,11 @@ def terminal(jobs_cmds: Optional[List[str]] = None, test_mode=False):
if first_time_user():
try:
- t_controller.call_intro(None)
+ # t_controller.call_intro(None)
+ webbrowser.open("https://docs.openbb.co/terminal/usage/basics")
# TDDO: Fix the CI
except EOFError:
pass
-
t_controller.print_help()
check_for_updates()
diff --git a/website/content/terminal/usage/basics/index.md b/website/content/terminal/usage/basics/index.md
index b492120fc8b4..67789c9d43c1 100644
--- a/website/content/terminal/usage/basics/index.md
+++ b/website/content/terminal/usage/basics/index.md
@@ -1,118 +1,316 @@
---
title: Basics
-description: The OpenBB Terminal is a modern investment research platform for everyone. It is a desktop application that allows you to access all the data and tools you need to make better investment decisions.
-keywords: [basics, installation, commands, menus, your own data, introduction, openbb terminal, explanation, basic usage, usage, how to, install, keyboard input]
+description: An introduction to the The OpenBB Terminal, a Command Line Interface (CLI) application. Functions (commands) are called through the keyboard with results returning as charts, tables, or text.
+keywords: [basics, commands, functions, features, menus, introduction, openbb terminal, obb, usage, how to, charts, tables, themes, styles, functions, data, sources, getting started]
---
+## Overview
+
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
-The OpenBB Terminal is based off the Command Line Interface (CLI)
-which is installed by default on every computer. By opening the application you have installed from the [Installation Page](/terminal/installation/windows),
-you are greeted with the following interface:
+The OpenBB Terminal is a Command Line Interface (CLI) application. Functions (commands) are called through the keyboard with results returned as charts, tables, or text. Charts and tables (if enabled) are displayed in a new window, and are fully interactive, while text prints directly to the Terminal screen. Commands are grouped into menus, with a menu or sub-menu being visually distinguishable from a function by the, `>`, on the far left of the screen. The color of the text can be altered under the [`/settings` menu](https://docs.openbb.co/terminal/usage/guides/customizing-the-terminal).
-
+Navigating through the Terminal menus is similar to following down a path, or traversing folders from any operating system's command line prompt. The `/home` screen is the main path where everything begins, and the menus are paths branched from the main. Instead of `C:\Users\OpenBB\Documents`, it is, [`/stocks/options`](https://docs.openbb.co/terminal/usage/intros/stocks/options). Instead of, `cd ..`, two periods - `..` - returns to the menu one level back towards the home screen.
-The OpenBB Terminal is centered around keyboard input. To navigate and perform analysis you will have to type in the name of the command followed by an `ENTER` (⏎). If you wish to see information about the OpenBB Terminal you can do so by typing `about` and then press `ENTER` (⏎). As you are typing, you will notice that you receive suggestions, by using the `DOWN` (⌄) arrow and pressing `ENTER` (⏎) you can select the command and execute it.
+:::note
+Absolute paths are also valid to-and-from any point. From the [`/stocks/options`](https://docs.openbb.co/terminal/usage/intros/stocks/options) menu, go directly to [`/crypto`](https://docs.openbb.co/terminal/usage/intros/crypto). By itself, `/`, returns to the home level.
+:::
-Throughout the entire terminal, the same set of colors are used which all share the same representation. This is structured as follows:
+
-- Light Blue: represents commands.
-- Dark Blue: represents menus, also recognizable by the `>` in front.
-- Orange: represents titles and headers.
-- Yellow: represents descriptions of a parameter or variable.
-- White: represents text, usually in combination with a description that is in Yellow.
+## Auto Complete
-### Explanation of Menus
+The OpenBB Terminal is equipped with an auto completion engine that presents choices based on the current menu. Whenever you start typing, the prompt will appear. When the function contains arguments, pressing the `space bar` after typing the command will present the list of available arguments. This functionality dramatically reduces the number of key strokes required to perform tasks and, in many cases, eliminates the need to consult the help dialogue for reminders. Choices - where they are bound by a defined list - are searchable with the up and down arrow keys.
-Menus, depicted in Dark Blue, take you to a new section of the terminal referred to as a menu. For example, if you wish to view information about stocks, you can do so by typing `stocks` and pressing `ENTER` (⏎). This opens a new menu as depicted below.
+
-
+## Help Dialogues
-Depending on the menu you are in, you are presented with a new set of commands and menus you can select. There are interactions in place between each menu. For example, when selecting a company within the `stocks` menu, the terminal will remember your selection when you visit the `fa` or `options` menu. See [Introduction to Stocks](/terminal/usage/intros/stocks).
+### -h or --help
-:::note **About Pipelines**
-You can quickly jump between menus by using a forward slash (`/`). For example, if I want to access the options menu, You can type `/stocks/options` to instantly arrive at this menu. You can do this from any location within the OpenBB Terminal!
-:::
+A help dialogue for any function at the current location is printed to the screen by typing `-h` after the command. The information returned contains a short description of the function and all accepted arguments. For an example, the `/news` function:
-### Explanation of Commands
+```console
+news -h
+```
-Commands, depicted in Light Blue, execute an action or task. For example,
-the commands that you are able to use from any menu in the terminal (see Explanation of Menus) are as follows:
+```console
+usage: news [-t TERM [TERM ...]] [-s SOURCES] [-h] [--export EXPORT] [--sheet-name SHEET_NAME [SHEET_NAME ...]] [-l LIMIT]
-- `cls`: clears the screen, by executing this command you are left with an empty screen.
-- `help`, `h` or `?`: displays the menu that you are currently on.
-- `quit`, `q` or `..`: allows for navigation through the menu. E.g. if you type `stocks` press `ENTER` (⏎) and then
- use `q` and press `ENTER` (⏎) you return to where you started. Validate this by typing `?` and pressing `ENTER` (⏎).
-- `support`: allows you to submit bugs, questions and suggestions.
-- `about`: this opens the related guide, linking to this website. It also has the ability to open a guide to a specific
- command. For example, within the `stocks` menu, `about candle` opens this guide.
-- `wiki`: search for a given expression on the Wikipedia without leaving the terminal.
+display news articles based on term and data sources
-Continuing with the example mentioned at `quit`, revisit the `stocks` menu and look at the commands. At the top you will see a command named load. To understand what this command can do, you can use `load -h` followed by `ENTER` (⏎). The `-h` stands for `help` and every command will have this feature. This will return the following:
+options:
+ -t TERM [TERM ...], --term TERM [TERM ...]
+ search for a term on the news
+ -s SOURCES, --sources SOURCES
+ sources from where to get news from (separated by comma)
+ -h, --help show this help message
+ --export EXPORT Export raw data into csv, json, xlsx
+ --sheet-name SHEET_NAME [SHEET_NAME ...]
+ Name of excel sheet to save data to. Only valid for .xlsx files.
+ -l LIMIT, --limit LIMIT
+ Number of entries to show in data.
+For more information and examples, use 'about news' to access the related guide.
```
-usage: load [-t TICKER] [-s START] [-e END] [-i {1,5,15,30,60}] [-p] [-f FILEPATH] [-m] [-w] [--exchange] [--performance] [-h] [--export EXPORT] [--sheet-name SHEET_NAME [SHEET_NAME ...]]
- [--source {YahooFinance,AlphaVantage,Polygon,EODHD,Intrinio}]
-
-Load stock ticker to perform analysis on. When the data source is yf, an Indian ticker can be loaded by using '.NS' at the end, e.g. 'SBIN.NS'. See available market in https://help.yahoo.com/kb/exchanges-data-
-providers-yahoo-finance-sln2310.html.
-
-optional arguments:
- -t TICKER, --ticker TICKER
- Stock ticker (default: None)
- -s START, --start START
- The starting date (format YYYY-MM-DD) of the stock (default: 2020-02-11)
- -e END, --end END The ending date (format YYYY-MM-DD) of the stock (default: 2023-02-15)
- -i {1,5,15,30,60}, --interval {1,5,15,30,60}
- Intraday stock minutes (default: 1440)
- -p, --prepost Pre/After market hours. Only reflected in 'YahooFinance' intraday data. (default: False)
- -f FILEPATH, --file FILEPATH
- Path to load custom file. (default: None)
- -m, --monthly Load monthly data (default: False)
- -w, --weekly Load weekly data (default: False)
- --exchange Show exchange information. (default: False)
- --performance Show performance information. (default: False)
- -h, --help show this help message (default: False)
- --export EXPORT Export raw data into csv, json, xlsx (default: )
- --sheet-name SHEET_NAME [SHEET_NAME ...]
- Name of excel sheet to save data to. Only valid for .xlsx files. (default: None)
- --source {YahooFinance,AlphaVantage,Polygon,EODHD,Intrinio}
- Data source to select from (default: YahooFinance)
-For more information and examples, use 'about load' to access the related guide.
+To search for news containing the term, "Federal Reserve", try this command:
+```console
+/news --term Federal Reserve
```
-This shows you all **arguments** the command has. These are additional options you can provide to the command. Each default value is also displayed which is used when you do not select this option. For example, if I would use the stock ticker of Amazon (AMZN, which can also be found with `search amazon`), I can use `load AMZN` which will return the following:
+### About
+
+`about` is a global function that opens a browser to the OpenBB documentation pages at the specific command or menu.
+```console
+/about stocks
```
-(🦋) /stocks/ $ load AMZN
-Loading Daily data for AMZN with starting period 2020-02-11.
+The command above will open a browser to [Introduction to the Stocks menu](https://docs.openbb.co/terminal/usage/intros/stocks).
-```
+### Support
-The default values you see within `load -h` have been inputted here. E.g. the starting period is 2020-02-11. I can decide to change these default values by calling the argument and inputting a different value.
+`support` is a global function for submitting a new request for support, a general question, or a bug report. The command will pre-populate a form with key information, like the command or menu name specific to the issue. Use the up and down arrow keys to browse and select the appropriate item for the ticket. Naturally, this command has a help dialogue.
-Whenever you wish to apply an optional argument, you use the related shortcode, e.g. `-s` or `--start`. Then, if there is an additional word behind the argument (in this case there is, which is `START`) it implies the argument expects you to define a value. Within the documentation you can read that the format must be `YYYY-MM-DD` implying that `2010-01-01` will be valid. If there is not an additional word behind it, it is enough to write down `load AMZN -p` (which refers to the prepost optional argument)
+```console
+support -h
+```
-Let's change the starting and ending period of the data that is being loaded in by doing the following:
+```console
+Submit your support request
+options:
+ -c {search,load,quote,tob,candle,news,resources,codes,ta,ba,qa,disc,dps,scr,sia,ins,gov,res,dd,fa,bt,ca,options,th,forecast}, --command {generic,search,load,quote,tob,candle,news,resources,codes,ta,ba,qa,disc,dps,scr,sia,ins,gov,res,dd,fa,bt,ca,options,th,forecast}
+ Command that needs support (default: None)
+ --msg MSG [MSG ...], -m MSG [MSG ...]
+ Message to send. Enclose it with double quotes (default: )
+ --type {bug,suggestion,question,generic}, -t {bug,suggestion,question,generic}
+ Support ticket type (default: generic)
+ -h, --help show this help message (default: False)
```
-(🦋) /stocks/ $ load AMZN -s 2005-01-01 -e 2010-01-01
-Loading Daily data for AMZN with starting period 2005-01-03.
+
+```console
+support search --type question --msg "How do I find stocks from India with OpenBB?"
```
-We can check that this period has changed by looking into the candle chart with `candle`. This, again shares the same `-h` argument. This results in the following which indeed depicts our selected period.
+
+
+The command opens a browser window to a pre-populated form on the OpenBB website. If you are signed-in to the Hub, all that is left to do is click `Submit`.
+
+
+
+**An answer to this question is**: `search --country india --exchange-country india`
+
+:::note
+Tips for submitting a support request:
+- Tell us what version number is installed.
+- Tell us what operating system and version the machine has.
+- What is the installation type? Installer, Source, PyPi, Docker, other?
+- Tell us the command and parameter combination causing the error.
+- Tell us what symbol (ticker) is, or was trying to be, loaded.
+- Show us the complete error message.
+- Let us know any contextual information that will help us replicate and accurately identify the problem.
+:::
+
+## Menus
+
+### The Main Menu
+
+The main menu, or the home screen, contains both menus and commands. Some of these commands are global, meaning they can be called from any location within the OpenBB Terminal. Refer to the in-depth introduction guides for each menu, for example, [Forecast](https://docs.openbb.co/terminal/usage/intros/forecast)
+
+| Function Key | Type | Is Global? | Description |
+| :----------- | :------: | :--------: | ------------------------------------------------------------------------------: |
+| about | Function | Yes | Opens a browser page to the documentation pages at the function or menu. |
+| account | Menu | - | Manage your OpenBB account. |
+| alternative | Menu | - | Alternative data sets. |
+| crypto | Menu | - | Digital assets. |
+| econometrics | Menu | - | Econometrics and custom data sets. |
+| economy | Menu | - | The broad economy. |
+| etf | Menu | - | Exchange-Traded Funds. |
+| exe | Function | No | Execute OpenBB Routine Scripts. |
+| featflags | Menu | - | Enable/disable Terminal behaviours. |
+| fixedincome | Menu | - | Central Bank and corporate bond rates and indexes. |
+| forecast | Menu | - | Time series forecasting and machine learning. |
+| forex | Menu | - | Currency pairs. |
+| funds | Menu | - | Mutual funds. |
+| futures | Menu | - | Commodity and financial futures. |
+| keys | Menu | - | Set and test API keys for data sources. |
+| intro | Function | No | An in-Terminal introduction. |
+| news | Function | No | Find news articles by term and source. |
+| portfolio | Menu | - | Portfolio and risk analysis. |
+| record | Function | Yes | Starts recording an OpenBB Routine Script. |
+| settings | Menu | - | Adjust Terminal settings. |
+| sources | Menu | - | Set preferred default sources. |
+| stop | Function | Yes | Stop recording the OpenBB Routine Script and save to the OpenBBUserData folder. |
+| support | Function | Yes | Report a bug or create a support ticket with OpenBB. |
+| survey | Function | No | Take a short user survey to help us improve. |
+| update | Function | No | Attempt to update (**Only for Github cloned repository installations**) |
+| wiki | Function | Yes | Query the Wikipedia API for a term or phrase. |
+
+### Additional Global Commands
+
+The commands listed below are not displayed on any Terminal menu, but are available from any location in the Terminal.
+
+| Function Key | Description |
+| :----------- | ------------------------------------------------: |
+| cls | Clears the Terminal screen. |
+| exit | Quits the Terminal. |
+| help, h, ? | Prints the current menu screen. |
+| quit, q, .. | Navigates back one menu towards Home. |
+| reset, r | Resets the Terminal, opening to the current menu. |
+
+## Data
+
+Many functions will require obtaining (free or subscription) API keys from various data providers. OpenBB provides methods for consuming these data feeds, but has no control over the quality or quantity of data provided to an end-user. **No API Keys are required to get started using the Terminal**. See the list of data providers [here](https://docs.openbb.co/terminal/usage/guides/api-keys), along with instructions for entering the credentials into the OpenBB Terminal. [Request a feature](https://openbb.co/request-a-feature) to let us know what we are missing!
+
+### Default Data Sources
+
+The default data source for each function (where multiple sources are available) can be defined within the [`/sources` menu](https://docs.openbb.co/terminal/usage/guides/changing-sources). The available sources for each function are displayed on the right of the menu, and they can be distinguished by the square brackets and distinct font color group. Unless a preference for a particular function is defined, the command will prioritize in the order they are displayed, from left-to-right, on the Terminal screen. To override a preference or default source, select one of the other choices by attaching the, `--source`, argument to the command syntax. The available sources for the feature will be populated by auto complete when the `space bar` is pressed after typing `--source`. This information is also printed with the `--help` dialogue of a command.
+
+```console
+/stocks/load AAPL/fa/income --source Polygon
```
-(🦋) /stocks/ $ candle
+
+
+
+### Importing and Exporting Data
+
+Most functions provide a method for exporting the raw data as a CSV, JSON, or XLSX file (with a specific sheet name). Exported data and user-supplied files to import are saved to the [OpenBBUserData folder](https://docs.openbb.co/terminal/usage/guides/data). The folder is located at the root of the operating system's User Account folder. Follow the link for a detailed description.
+
+## Charts
+
+The OpenBB charting library provides interactive, customizable, charts. Here's an example of displaying weekly candles for AAPL.
+
+```console
+/stocks/load AAPL -w/candle
```
-
+
+
+### Toolbar
+
+The toolbar is located at the bottom of the window, and provides methods for:
+
+- Panning and zooming.
+- Modifying the title and axis labels.
+- Adjusting the hover read out.
+- Toggling light/dark mode.
+- Annotating and drawing.
+- Exporting raw data.
+- Saving the chart as an image.
+- Adding supplementary external data as an overlay.
+
+The label for each tool is displayed by holding the mouse over it.
+
+
+
+Toggle the toolbar's visibility via the keyboard with, `ctrl + h`.
+
+### Text Tools
+
+Annotate a chart by clicking on the `Add Text` button, or with the keyboard, `ctrl + t`.
+
+
+
+Enter some text, make any adjustments to the options, then `submit`. Place the crosshairs over the desired data point and click to place the text.
+
+
+
+After placement, the text can be updated or deleted by clicking on it again.
+
+
+
+The title of the chart is edited by clicking the button, `Change Titles`, near the middle center of the toolbar, immediately to the right of the `Add Text` button.
+
+### Draw Tools
+
+The fourth group of icons on the toolbar are for drawing lines and shapes.
+
+- Edit the colors.
+- Draw a straight line.
+- Draw a freeform line.
+- Draw a circle.
+- Draw a rectangle.
+- Erase a shape.
+
+To draw on the chart, select one of the four drawing buttons and drag the mouse over the desired area. Click on any existing shape to modify it by dragging with the mouse and editing the color, or remove it by clicking the toolbar button, `Erase Active Shape`. The edit colors button will pop up as a floating icon, and clicking on that will display the color palette.
+
+
+
+### Export Tools
+
+The two buttons at the far-right of the toolbar are for saving the raw data or, to save an image file of the chart at the current panned and zoomed view.
+
+
+
+### Overlay
+
+The button, `Overlay chart from CSV`, provides an easy import method for supplementing a chart with additional data. Clicking on the button opens a pop-up dialogue to select the file, column, and whether the overlay should be a bar, candlestick, or line chart. As a candlestick, the CSV file must contain OHLC data. The import window can also be opened with the keyboard, `ctrl-o`.
+
+
+
+After choosing the file to overlay, select what to show and then click on `Submit`.
+
+
+
+
+
+### Cheat Sheet
+
+The image below can be saved and used as a reference.
+
+
+
+
+## Tables
+
+The OpenBB Terminal sports interactive tables which opens in a separate window. They provide methods for searching, sorting, filtering, and exporting directly within the table. Preferences and settings for the tables can be updated directly on the table.
+
+### Sorting and Filtering
+
+Columns can be sorted ascending/descending/unsorted, by clicking the controls to the right of each header title. The status of the filtering is shown as a blue indicator.
+
+
+
+The settings button, at the lower-left corner, displays choices for customizing the table. By selecting the `Type` to be `Advanced`, columns become filterable.
+
+
+
+The columns can be filtered with min/max values or by letters, depending on the content of each column.
+
+
+
+### Selecting Columns and Rows
+
+The table will scroll to the right as far as there are columns. Columns can be removed from the table by clicking the icon to the right of the settings button and unchecking it from the list.
+
+
+
+The number of rows per page is defined in the drop down selection near the center, at the bottom.
+
+
+
+### Freeze the Index and Column Headers
+
+Right-click on the index name to enable/disable freezing when scrolling to the right. Column headers are frozen by default.
+
+
+
+### Exporting Data
+
+At the bottom-right corner of the table window, there is a button for exporting the data. To the left, the drop down selection for `Type` can be defined as a CSV, XLSX, or PNG file. Exporting the table as a PNG file will create a screenshot of the table at its current view, and data that is not on the screen will not be captured.
+
+
+
+### Cheat Sheet
+
+The image below can be saved and used as a reference.
-As mentioned in the Explanation of Menus, some information also transfers over to other menus and this includes the loaded market data from load. So, if you would visit the `ta` menu (which stands for Technical Analysis) you will see that, by running any command, the selected period above is depicted again. Return to the Stocks menu again by using `q` and use it again to return to the home screen which can be shown with `?`.
+
diff --git a/website/content/terminal/usage/guides/customizing-the-terminal.md b/website/content/terminal/usage/guides/customizing-the-terminal.md
index f578954edf53..6705f61079b4 100644
--- a/website/content/terminal/usage/guides/customizing-the-terminal.md
+++ b/website/content/terminal/usage/guides/customizing-the-terminal.md
@@ -1,63 +1,140 @@
---
sidebar_position: 5
title: Customization
-description: Learn how to customize the look of your OpenBB Terminal. You can adjust the lay-out, settings, terminal behavior, and enable or disable certain functionalities.
-keywords: [settings, featflags, feature flags, lay-out, advanced, customizing, openbb terminal, layout, settings, colors, functionality, features]
+description: The settings and feature flags menus are for altering the behaviour and presentation of the Terminal, both are accessed from the main menu.
+keywords: [settings, featflags, feature flags, lay-out, advanced, customizing, openbb terminal]
---
+The OpenBB Terminal contains two menus for altering the behaviour and presentation of the Terminal, Settings and FeatFlags, both of which are accessed from the main menu.
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
+## The Settings Menu
-Since the code is open source you are able to adjust anything you like. However, to make it easy for you we have created a settings and featflags menu that include the most requested features that users like to have control over.
-
-## Using the Settings Menu
-
-To adjust the lay-out and settings of the OpenBB Terminal you can access the `settings` menu. This menu allows you to tweak how the terminal behaves.
-
-
-
-This menu includes the following:
-
-- `colors` define the colors you wish to use within the OpenBB Terminal.
-- `dt` adds or removes the datetime from the flair (which is next to the flair).
-- `flair` allows you to change the emoji that is used.
-- `lang` gives the ability to change the terminal language. At this moment, the terminal is only available in English.
-- `userdata` defines the folder you wish to export data you acquire from the terminal. Use quotes for custom locations.
-- `tz` allows you to change the timezone if this is incorrectly displayed for you.
-- `autoscaling` automatically scales plots for you if enabled (when green).
- - if `autoscaling` is enabled:
- - `pheight` sets the percentage height of the plot (graphs) displayed (if autoscaling is enabled).
- - `pwidth` sets the percentage width of the plot (graphs) displayed (if autoscaling is enabled).
- - if `autoscaling` is disabled:
- - `height` sets the height of the plot (graphs) displayed (if autoscaling is disabled).
- - `width` sets the width of the plot (graphs) displayed (if autoscaling is disabled).
-- `dpi` refers to the resolution that is used for the plot (graphs)
-- `backend` allows you to change the backend that is used for the graphs
-- `monitor` choose which monitors to scale the plot (graphs) to if applicable
-- `source` allows you to select a different source file in which the default data sources are written down
-
-## Using the Feature Flags Menu
-
-To enable or disable certain functionalities of the terminal you can use the `featflags` menu.
-
-
-
-By entering one of the commands on this page you are able to turn the feature flag on or off. This menu includes the following:
-
-- `retryload` whenever you misspell commands, try to use the `load` command with it first (default is off).
-- `tab` whether to use tabulate to print DataFrames, to prettify these DataFrames (default is on).
-- `cls` whether to clear the command window after each command (default is off).
-- `color` whether to use colors within the terminal (default is on).
-- `promptkit` whether you wish to enable autocomplete and history (default is on).
-- `thoughts` whether to receive a thought of the day (default is off).
-- `reporthtml` whether to open reports as HTML instead of Jupyter Notebooks (default is on).
-- `exithelp` whether to automatically print the help message when you use `q` (default is off).
-- `rcontext` whether to remember loaded tickers and similar while switching menus (default is on).
-- `rich` whether to apply a colorful rich terminal (default is on).
-- `richpanel` whether to apply a colorful rich terminal panel (default is on).
-- `ion` whether to enable interactive mode of MATPLOTLIB (default is on).
-- `watermark` whether to include the watermark of OpenBB Terminal in figures (default is on).
-- `cmdloc` whether the location of the command is displayed in figures (default is on).
-- `tbhint` whether usage hints are displayed in the bottom toolbar (default is on).
-- `overwrite` whether to automatically overwrite Excel files if prompted to (default is off).
+The `/settings` menu provides methods for customizing the look of the Terminal. Enter the menu from anywhere in the Terminal with:
+
+```console
+/settings
+```
+
+| Function Key | Description |
+| :----------- | ---------------------------------------------------------------: |
+| chart | Select the chart style. |
+| colors | Sets the color scheme for Terminal fonts. |
+| dt | Add or remove date and time from the Terminal command line. |
+| flair | Sets the flair emoji to be used. |
+| height | Set the default plot height. |
+| lang | Select the language for the Terminal menus and commands. |
+| source | Use an alternate data sources file. (Not recommended to change.) |
+| table | Select the table style. |
+| tz | Select a timezone. |
+| userdata | Change the local path to the OpenBBUserData folder. |
+| width | Set the default plot width. |
+
+### Examples
+
+#### Styles
+
+Set charts and tables styles as light or dark mode.
+
+```console
+/settings/table -s light
+```
+
+```console
+/settings/chart -s dark
+```
+
+#### tz
+
+Set the local timezone for the Terminal
+
+```console
+
+/settings/tz Africa/Johannesburg
+```
+
+## The Feature Flags Menu
+
+The `/featflags` menu provides methods for altering the behaviour and responses with environment variables. These configurations are on/off, and the status is indicated by the red/green text of each. Each parameter is listed below.
+
+| Function Key | Description |
+| :----------- | ----------------------------------------------------------------------------------------------: |
+| cls | Clear the screen after each command. Default state is off. |
+| exithelp | Automatically print the screen after navigating back one menu. Default state is off. |
+| interactive | Enable/disable interactive tables. Disabling prints the table directly on the Terminal screen. |
+| overwrite | Automatically overwrite exported files with the same name. Default state is off. |
+| promptkit | Enable auto complete and history. Default state is on. |
+| rcontext | Remember loaded tickers while switching menus. Default state is on. |
+| retryload | Retries misspelled commands with the load function first. Default state is off. |
+| reporthtml | Generate reports as HTML files. Default state is on. |
+| richpanel | Displays a border around menus. Default state is on. |
+| tbhint | Display usage hints in the bottom toolbar. Default state is on. |
+| version | Displays the currently installed version number in the bottom right corner. |
+
+### Examples
+
+#### interactive
+
+When it is off, the Terminal displays all tables directly on the screen instead of opening a window.
+
+```console
+(🦋) /stocks/ $ quote spy
+
+ SPY Quote
+┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃ Info ┃ Value ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ Symbol │ SPY │
+├───────────────────────┼────────────────────────┤
+│ Name │ SPDR S&P 500 ETF Trust │
+├───────────────────────┼────────────────────────┤
+│ Price │ 412.43 │
+├───────────────────────┼────────────────────────┤
+│ Changes percentage │ 0.06 │
+├───────────────────────┼────────────────────────┤
+│ Change │ 0.23 │
+├───────────────────────┼────────────────────────┤
+│ Day low │ 410.60 │
+├───────────────────────┼────────────────────────┤
+│ Day high │ 413.06 │
+├───────────────────────┼────────────────────────┤
+│ Year high │ 431.73 │
+├───────────────────────┼────────────────────────┤
+│ Year low │ 348.11 │
+├───────────────────────┼────────────────────────┤
+│ Market cap │ 378.521 B │
+├───────────────────────┼────────────────────────┤
+│ Price avg50 │ 402.47 │
+├───────────────────────┼────────────────────────┤
+│ Price avg200 │ 394.88 │
+├───────────────────────┼────────────────────────┤
+│ Exchange │ AMEX │
+├───────────────────────┼────────────────────────┤
+│ Volume │ 44.621 M │
+├───────────────────────┼────────────────────────┤
+│ Avg volume │ 89774263 │
+├───────────────────────┼────────────────────────┤
+│ Open │ 411.99 │
+├───────────────────────┼────────────────────────┤
+│ Previous close │ 412.20 │
+├───────────────────────┼────────────────────────┤
+│ Eps │ 19.85 │
+├───────────────────────┼────────────────────────┤
+│ Pe │ 20.78 │
+├───────────────────────┼────────────────────────┤
+│ Earnings announcement │ 2017-11-29 17:00:00 │
+├───────────────────────┼────────────────────────┤
+│ Shares outstanding │ 917.782 M │
+├───────────────────────┼────────────────────────┤
+│ Timestamp │ 2023-04-24 12:34:22 │
+└───────────────────────┴────────────────────────┘
+```
+
+#### overwrite
+
+Enable this feature flag to remove the prompt when exporting a file with the same name. This will only overwrite an existing `XLSX` file if the `--sheet-name` is not defined.
+
+#### exithelp
+
+Enabling this prints the parent menu on the screen when going back from a sub-menu.
diff --git a/website/content/terminal/usage/intros/common/ta.md b/website/content/terminal/usage/intros/common/ta.md
index a0cf8eedd28d..436dbf28e7df 100644
--- a/website/content/terminal/usage/intros/common/ta.md
+++ b/website/content/terminal/usage/intros/common/ta.md
@@ -1,7 +1,7 @@
---
title: Technical Analysis
-keywords: [technical, analysis, ta, t/a, intraday, daily, indicators, signals, average, moving, exponential, rsi, fibonacci, retracement, bollinger, heltner, accumulation, distribution, obv, on-balance, volume, volatility, trend, momentum, overlap, crypto, stocks, funds, etf, etfs, example, how to, how to use, fibonacci]
-description: This guide introduces the Technical Analysis menu, which is common across many sections of the OpenBB Terminal.
+keywords: [technical, analysis, ta, t/a, intraday, daily, indicators, signals, average, moving, exponential, rsi, fibonacci, retracement, bollinger, heltner, accumulation, distribution, obv, on-balance, volume, volatility, trend, momentum, overlap, crypto, stocks, funds, etf, etfs, realized, fx, forex]
+description: An introduction to the Technical Analysis menu of the OpenBB Terminal. This guide provides an overview of the functions and provides examples for use.
---
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
@@ -10,10 +10,10 @@ import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
The Technical Analysis menu offers the user a suite of tools for analyzing the technical components of an asset's trading history. The menu can be found in most wings of the Terminal:
-- Crypto
-- Stocks
-- ETF
-- Forex
+- [Crypto](https://docs.openbb.co/terminal/usage/intros/crypto)
+- [Stocks](https://docs.openbb.co/terminal/usage/intros/stocks)
+- [ETF](https://docs.openbb.co/terminal/usage/intros/etf)
+- [Forex](https://docs.openbb.co/terminal/usage/intros/forex)
The commands are divided by categories that define their purpose for general-use:
@@ -22,76 +22,235 @@ The commands are divided by categories that define their purpose for general-use
- Trend - Directional strength
- Volatility - Width of the price bands
- Volume - Singling out volume
-- Custom - Fibonacci retracements
+- Custom - Multiple indicators and Fibonacci retracements
-All commands in this menu will rely on the interval and window chosen when loading an asset for analysis. Refer to the directory tree on the left side of the page for information on individual commands. To get a better understanding of what these features are, and the formulas behind them, a number of sources should be consulted; but, a good starting point is Investopedia.
+All commands in this menu will rely on the interval and window chosen when loading an asset for analysis. Refer to the directory tree on the left side of the page, [here](https://docs.openbb.co/terminal/reference), for information on individual commands. To get a better understanding of what these features are, and the formulas behind them, a number of sources should be consulted; but, a good starting point is [Investopedia](https://www.investopedia.com/terms/t/technicalanalysis.asp). The menu employs the [Pandas-TA Library](https://github.com/twopirllc/pandas-ta). Submit a [feature request](https://openbb.co/request-a-feature) to let us know which indicators we should add next!
-### How to use
+### How to Use
-To begin, enter the menu from one of the menus listed above by entering `ta`.
+To begin, enter the menu from one of the menus listed above by entering `ta`. For demonstration purposes, we will use `QQQ` as the ticker. Let's grab some data!
-
+```console
+/stocks/load QQQ/ta
+```
+
+The block above loads daily QQQ historical prices and volume, then enters the Technical Analysis menu. The table below lists all the available analysis functions.
+
+| Function Key | Type | Description |
+| :----------- | :--------- | ------------------------------------------: |
+| load | - | Load a new ticker for analysis. |
+| ema | Overlap | Exponential Moving Average. |
+| hma | Overlap | Hull Moving Average. |
+| sma | Overlap | Simple Moving Average. |
+| wma | Overlap | Weighted Moving Average. |
+| vwap | Overlap | Volume Weighted Average Price. |
+| zlma | Overlap | Zero-Lag Moving Average. |
+| cci | Momentum | Commodity Channel Index. |
+| cg | Momentum | Center of Gravity. |
+| clenow | Momentum | Clenow Volatility Adjusted Momentum. |
+| demark | Momentum | Tom Demark's Sequential Indicator. |
+| macd | Momentum | Moving Average Convergence/Divergence. |
+| fisher | Momentum | Fisher Transform. |
+| rsi | Momentum | Relative Strength Index. |
+| rsp | Momentum | Relative Strength Percentile. |
+| stoch | Momentum | Stochastic Oscillator. |
+| adx | Trend | Average Directional Movement Index. |
+| aroon | Trend | Aroon Indicator. |
+| atr | Volatility | Average True Range. |
+| bbands | Volatility | Bollinger Bands. |
+| cones | Volatility | Realized Volatility Cones. |
+| donchian | Volatility | Donchian Channels. |
+| kc | Volatility | Keltner Channels. |
+| ad | Volume | Accumulation/Distribution Line. |
+| adosc | Volume | Chaikin Oscillator. |
+| obv | Volume | On-Balance Volume. |
+| fib | Custom | Fibonacci Retracement. |
+| multi | Custom | Plot multiple indicators on the same chart. |
+
+:::note
-Choose a beginning and end date to see a volume-weighted average price chart of the loaded ticker. `vwap --start 2022-01-01 --end 2022-06-17`
+Some functions will not be compatible with intraday data and some may be designed specifically for a daily window. Interval labels on charts may still be described as "days" when the time series interval is intraday. Consult the help dialogue, by attaching `-h` to a command, for a reminder of the adjustable parameters.
+:::
-
+With some daily data now loaded, let's look at some charts!
-The Fibonacci retracements are drawn with `fib`
+## Examples
-
+### Moving Averages
-See the on-balance volume for the time-period loaded.
+There are five types of moving averages available, they are grouped into the `Overlay` category. It is possible to overlay multiple windows of time for each one, and it is also possible to overlay multiple versions of moving average.
-
+#### Single MA Type
-The help dialogue for any feature is printed by attaching `-h` to the command.
+The help dialogue will explain the differences. For example, Zero-Lag Moving Average.
-### Examples
+```console
+zlma -h
+```
+
+```console
+usage: zlma [-l N_LENGTH] [-o N_OFFSET] [-h] [--export EXPORT] [--sheet-name SHEET_NAME [SHEET_NAME ...]]
-`recom` & `summary` are commands available only with a stock loaded as the asset. `summary` is a text description of the technical conditions.
+The zero lag exponential moving average (ZLEMA) indicator was created by John Ehlers and Ric Way. The idea is do a regular exponential moving average (EMA) calculation but
+on a de-lagged data instead of doing it on the regular data. Data is de-lagged by removing the data from "lag" days ago thus removing (or attempting to) the cumulative
+effect of the moving average.
+options:
+ -l N_LENGTH, --length N_LENGTH
+ Window lengths. Multiple values indicated as comma separated values. (default: [20])
+ -o N_OFFSET, --offset N_OFFSET
+ offset (default: 0)
+ -h, --help show this help message (default: False)
+ --export EXPORT Export raw data into csv, json, xlsx and figure into png, jpg, pdf, svg (default: )
+ --sheet-name SHEET_NAME [SHEET_NAME ...]
+ Name of excel sheet to save data to. Only valid for .xlsx files. (default: None)
+
+For more information and examples, use 'about zlma' to access the related guide.
```
-(🦋) /stocks/ta/ $ summary
-MSFT price has changed 1.29% in the last 3 days and 1.09% yesterday.
-RSI is less than 30 and the indicator is pointing downwards.
-Asset is in the oversold area.
-MACD is in the Bearish area and the histogram is moving upwards.
-MSFT price is trading below the 200-day SMA line and the SMA is trending down.
-The asset price is between the Middle and the Lower Bollinger Bands.
+
+For moving averages, there are two parameters: the window length and offset. Let's examine the ZLMA at 50 and 200 days.
+
+```console
+zlma -l 50,200
```
-`recom` projects buy & sell signals for the short-term.
+
+
+Drawing a chart with the Simple Moving Average shows the overlaps occur at different points in time.
+```console
+sma -l 50,200
```
-(🦋) /stocks/ta/ $ recom
-
- Ticker Recommendation
-┏━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━┳━━━━━━┳━━━━━━━━━┓
-┃ ┃ RECOMMENDATION ┃ BUY ┃ SELL ┃ NEUTRAL ┃
-┡━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━╇━━━━━━╇━━━━━━━━━┩
-│ 1 month │ SELL │ 2 │ 15 │ 9 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 1 week │ SELL │ 5 │ 11 │ 10 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 1 day │ BUY │ 11 │ 5 │ 10 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 4 hours │ BUY │ 15 │ 2 │ 9 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 1 hour │ BUY │ 14 │ 3 │ 9 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 15 min │ BUY │ 15 │ 3 │ 8 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 5 min │ BUY │ 16 │ 2 │ 8 │
-├─────────┼────────────────┼─────┼──────┼─────────┤
-│ 1 min │ BUY │ 16 │ 3 │ 7 │
-└─────────┴────────────────┴─────┴──────┴─────────┘
+
+
+
+#### Multiple MA Types
+
+Let's overlay the 200-day ZLMA with the 200-day SMA to see where they intersect. This is accomplished using the `multi` function.
+
+```console
+multi -i sma[200],zlma[200]
```
-Bollinger Bands with a 1-minute resolution for AMZN - `bbands`
+:::note
+Make note of the difference in parameters syntax. With the `multi` function, parameters for each indicator must be surrounded with square brackets [ ].
+:::
+
+
+
+Now let's see both 200 and 50-day moving averages.
-
-
+```console
+multi -i sma[50,200],zlma[50,200]
+```
+
+
+
+The last crossover point provides some confirmation of the current trend. Intraday data might reveal more. Let's see the one-hour MAs!
+
+```console
+load qqq -i 60/multi -i sma[50,200],zlma[50,200]
+```
+
+
+
+The ZLMA 50 has crossed over the SMA50, potentially signalling that the trend is near its exhaustion point. Let's consult some other indicators using the hourly data now loaded.
+
+### multi
+
+Load multiple indicators on the same chart with the `multi` command.
+
+```console
+load qqq -i 60 -s 2023-01-01/multi rsi,vwap,atr
+```
+
+
+
+The help dialogue for this function provides some guidance for setting the optional arguments for each indicator.
+
+```console
+multi -h
+```
+
+```console
+usage: multi [-i INDICATORS] [-h] [--export EXPORT] [--sheet-name SHEET_NAME [SHEET_NAME ...]]
+
+Plot multiple indicators on the same chart separated by a comma.
+
+options:
+ -i INDICATORS, --indicators INDICATORS
+ Indicators with optional arguments in the form of "macd[12,26,9],rsi,sma[20]" (default: None)
+ -h, --help show this help message (default: False)
+ --export EXPORT Export raw data into csv, json, xlsx and figure into png, jpg, pdf, svg (default: )
+ --sheet-name SHEET_NAME [SHEET_NAME ...]
+ Name of excel sheet to save data to. Only valid for .xlsx files. (default: None)
+
+For more information and examples, use 'about multi' to access the related guide.
+```
+
+Making adjustments with this function requires knowing the parameters for each individual indicator. They will have sensible default values but, where there are multiple parameters, it may not be obvious which order the numbers need to be entered. Sometimes they are not even numbers, like MACD.
+
+### macd
+
+```console
+macd --help
+
+(🦋) /stocks/ta/ $ macd --help
+
+usage: macd [--fast N_FAST] [--slow N_SLOW] [--signal N_SIGNAL] [-h] [--export EXPORT] [--sheet-name SHEET_NAME [SHEET_NAME ...]]
+
+The Moving Average Convergence Divergence (MACD) is the difference between two Exponential Moving Averages. The Signal line is an Exponential Moving
+Average of the MACD. The MACD signals trend changes and indicates the start of new trend direction. High values indicate overbought conditions, low
+values indicate oversold conditions. Divergence with the price indicates an end to the current trend, especially if the MACD is at extreme high or low
+values. When the MACD line crosses above the signal line a buy signal is generated. When the MACD crosses below the signal line a sell signal is
+generated. To confirm the signal, the MACD should be above zero for a buy, and below zero for a sell.
+
+options:
+ --fast N_FAST The short period. (default: 12)
+ --slow N_SLOW The long period. (default: 26)
+ --signal N_SIGNAL The signal period. (default: 9)
+ -h, --help show this help message (default: False)
+ --export EXPORT Export raw data into csv, json, xlsx and figure into png, jpg, pdf, svg (default: )
+ --sheet-name SHEET_NAME [SHEET_NAME ...]
+ Name of excel sheet to save data to. Only valid for .xlsx files. (default: None)
+
+For more information and examples, use 'about macd' to access the related guide.
+```
+
+The one-hour MACD generated a downward signal trend two days ago.
+
+```console
+load qqq -i 60 -s 2023-01-01/macd
+```
+
+
+
+:::note
+Clicking and dragging the mouse near the corners at each axis allows the zooming to be locked to the X or Y axis only.
+:::
+
+### obv
+
+Looking at the one-minute on-balance volume of QQQ today (April 21, 2023) reveals that massive volume spike occurred at 11:04. The ceiling is now the floor.
+
+```console
+load qqq -i 1 -s 2023-04-21/macd
+```
+
+
+
+The Accumulation/Distribution Line at the same one-minute interval signals in advance of the upward drift reversal, beginning to sell into the Friday close just before 14:00.
+
+
+
+## Indicators Dashboard
+
+This menu is also available as an experimental Dashboard Streamlit App.
+
+```console
+/dashboards/indicators
+```
-The Accumulation/Distribution line of AMZN - `ad`
+
-
+
diff --git a/website/content/terminal/usage/intros/forecast.md b/website/content/terminal/usage/intros/forecast.md
index a0ff8d16d17e..9aed1f73bd40 100644
--- a/website/content/terminal/usage/intros/forecast.md
+++ b/website/content/terminal/usage/intros/forecast.md
@@ -1,7 +1,7 @@
---
title: Forecast
-keywords: [machine learning, statistics, artificial intelligence, deep learning, neural networks, time series, research, feature engineering, hyperparameters, ai, ml, openbb terminal, how to, example, workflow]
-description: A brief guide to the Forecast menu. This introduction reviews the capabilities, functionality, commands, outputs, and examples in context.
+keywords: [machine learning, statistics, artificial intelligence, deep learning, neural networks, time series, research, feature engineering, hyperparameters, ai, ml, openbb terminal, darts, u8, u8darts, pytorch, regression, linear, models]
+description: An introduction to the OpenBB Terminal's Forecast menu and machine learning toolkit.
---
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
@@ -10,191 +10,351 @@ import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
The Forecast menu is a machine learning toolkit that provides practitioners with high-level components that can quickly provide state-of-the-art results, be it with with classical or deep learning models, while also providing researchers with low-level components that can be mixed, matched and fine tuned to build new approaches and custom tuned models. Bring in multiple datasets and train machine learning models with unlimited external factors to see how underlying data may change future forecasting predictions and accuracy.
-### Accessing the Forecast menu
+## The Forecast Menu
-The Forecast menu is called upon by typing `forecast` which opens the following menu:
+The Forecast menu can be called from any location with the command:
+```console
+ /forecast
```
-(🦋) / $ forecast
-```
-
+
+
+There are also methods for entering the menu with a loaded ticker symbol from either of the [`/crypto` menu](https://docs.openbb.co/terminal/usage/intros/crypto) and [`/stocks` menu](https://docs.openbb.co/terminal/usage/intros/stocks)
+
+The menu is divided into sections for:
+
+- Loading Data
+- Data Exploration
+- Feature Engineering
+- Time Series Forecasting
+- Anomaly Detection
+- Miscellaneous AI Tools
+
+and the functions within these groups are described in the following sections.
+
+### Loading Data
+
+#### With the Load Command
+
+If the Forecast menu has not been entered directly through the [`/crypto`](https://docs.openbb.co/terminal/usage/intros/crypto) or [`/stocks`](https://docs.openbb.co/terminal/usage/intros/stocks) menus, a dataset must be loaded before commencing any work. Use the `load` command to open one from a CSV file placed in the OpenBBUserData folder. The paths where the auto completion engine is looking for files is printed on the screen directly above the `load` command, `Looking for data in:`
-### How to Use
+
-To begin any machine learning, you must first load in data. The menu supports importing both terminal datasets found in `stocks` and `cryptocurrency`, along with external datasets in the form of `.csv` that can be placed in the following location: `~/OpenBBUserData/custom_imports/forecast/`
+Use the following syntax to load a file and name a dataset, substituting `vix_daily` with the name of the file.
-It is important to note, if you ever have troubles on running a command, please read the help commands to guide you on what is available. You can do this by trailing any command with a `-h`
+```console
+load --file vix_daily.csv --alias vix
```
-(🦋) /forecast/ $ -h
+
+To refresh the screen, enter: `?`
+
+The dataset will be listed under the `load` command and display the column names within it.
+
+
+
+Repeat the process until all desired datasets have been loaded.
+
+#### Via the Stocks/Crypto Menus
+
+Use the `load` command, according to the nuances of each menu. For example's sake, and to match the starting date of the previously loaded file, it will look like:
+
+```console
+/stocks/load SPY --start 1997-01-01/forecast
```
-#### Loading Data
+Combine these two methods an unlimited number of times for building a more complex model and forecast.
-Importing data from within the terminal can be simply done as follows:
+
+With some data loaded, the first series of tools are for inspecting, managing, and exporting the sets.
+
+### Exploration
+
+The exploration functions are listed in the table below, along with a short description.
+
+|Function |Description |
+|:--------|-----------:|
+|clean |Clean a dataset by filling or dropping NaNs. |
+|combine |Combine columns from different datasets. |
+|corr |Plot the correlation coefficients for dataset features. |
+|delete |Delete columns from dataset. |
+|desc |Show descriptive statistics of a dataset. |
+|export |Export a processed dataset. |
+|plot |Plot a specific columns of a loaded dataset. |
+|rename |Rename columns from dataset. |
+|season |Plot the seasonality for a dataset column. |
+|setndays |Set the default number of days to forecast. |
+|show |Show a portion of a loaded dataset. |
+
+#### Combine
+
+When running a model, all targeted columns must be within the same dataset. Use the `combine` command to accomplish this by merging one column with another dataset.
+
+```console
+combine --dataset SPY -c vix.high
+combine --dataset SPY -c vix.low
```
-(🦋) / $ stocks
-(🦋) /stocks/ $ load AAPL
-(🦋) /stocks/ $ forecast
+
+The SPY time series now has two additional columns from the VIX dataframe, high and low.
+
+#### Delete
+
+It may be desirable to remove entire columns from the target dataset. The syntax below removes the three columns from the SPY data loaded through the stocks menu.
+
+```console
+delete --delete SPY.adj_close
+delete --delete SPY.stock_splits
+delete --delete SPY.dividends
```
-The menu can support loading in unlimited datasets. Once the first one is loaded, the menu options should turn blue to allow you to begin working through the menu options.
+#### Show
+
+After performing operations, examine the results using the `show` command.
-To load external data in the form as `.csv`, please place them into the folder shown within your terminal menu and then load them in as follows:
+
+```console
+show --name SPY
```
-(🦋) / $ forecast
-(🦋) /forecast/ $ load msft.csv
-(🦋) /forecast/ $ load btc.csv
+
+
+
+#### Export
+
+To save all of the combined changes within a dataset, export it to a new file.
+
+```console
+export -d SPY --type csv
```
-
+This creates a new file in the [OpenBBUserData folder](https://docs.openbb.co/terminal/usage/guides/data).
-#### Understanding the structure of the menu
+### Feature Engineering
-The menu is broken up into three distinct sections. These sections represent the typical order of operations a machine learning professional would have to take in order to complete a project. Once data is loaded in, begin with `Exploration`.
+The Feature Engineering section provides methods for making fast calculations with a dataset. Each one is listed below with a short description.
-##### Exploration
-Explore the datasets loaded into the menu to further understand your data and create unique new datasets by combining and analyzing features. Functions to note: `plot`, `combine`, `desc`, `corr`.
+|Function |Description |
+|:--------|-----------:|
+|atr |Add Average True Range |
+|delta |Add % Change |
+|ema |Add Exponentially Weighted Moving Average |
+|mom |Add Momentum |
+|roc |Add Rate of Change |
+|rsi |Add Relative Strength Index |
+|sto |Add Stochastic Oscillator %K and %D |
+|signal |Add Price Signal (short vs. long term) |
-
+Each function will have slight variations to the command syntax, but will generally operate similarly. Print the help dialogue as a reminder.
-##### Feature Engineering
-Manuipulate datasets (addition, deletion, combination, mutation) of your data set to potentially improve machine learning model training, by providing new features and covariates that may leading to better performance and greater accuracy.
+```console
+usage: rsi [-d {vix,SPY}] [-c TARGET_COLUMN] [--period PERIOD] [-h]
-
+Add rsi to dataset based on specific column.
-##### imeSeries Forecasting
-Train state of the art models on custom datasets and experiment tuning hyperparameters. For more information on specific model
-implementations, please see [Darts Models](https://unit8co.github.io/darts/generated_api/darts.models.forecasting.html)
-for in depth documentation.
-
+options:
+ -d {vix,SPY}, --dataset {vix,SPY}
+ The name of the dataset you want to select (default: None)
+ -c TARGET_COLUMN, --target-column TARGET_COLUMN
+ The name of the specific column you want to use (default: close)
+ --period PERIOD The period to use (default: 10)
+ -h, --help show this help message (default: False)
-### Sample workflow #1 (beginner)
-Let's begin by using one of the datasets we loaded in previously : `AAPL`
+For more information and examples, use 'about rsi' to access the related guide.
+```
-We will be forecasting `5 Business days` ahead for the remaider of these workflows unless specified.
+```console
-**Note:** All models automatically perform Historical backtesting on the test split before providing a prediction.
+usage: atr [--close-col CLOSE_COL] [--high-col HIGH_COL] [--low-col LOW_COL] [-d {vix,SPY}]
+ [-c TARGET_COLUMN] [-h]
-We use [MAPE](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error) as it is quite convenient and scale independent since it calculates error as a percentage instead of an absolute value. THere are many more metrics to compare time series. The metrics will compare only common slices of series when the two series are not aligned, and parallelize computation over a large number of pairs of series. More metrics will be released in future versions of the menu.
+Add Average True Range to dataset of specific stock ticker.
+options:
+ --close-col CLOSE_COL
+ Close column name to use for Average True Range. (default: close)
+ --high-col HIGH_COL High column name to use for Average True Range. (default: high)
+ --low-col LOW_COL Low column name to use for Average True Range. (default: low)
+ -d {vix,SPY}, --dataset {vix,SPY}
+ The name of the dataset you want to select (default: None)
+ -c TARGET_COLUMN, --target-column TARGET_COLUMN
+ The name of the specific column you want to use (default: close)
+ -h, --help show this help message (default: False)
+
+For more information and examples, use 'about atr' to access the related guide.
```
-(🦋) /forecast/ $ plot AAPL.close
+
+To use these commands with the default settings, apply the syntax below.
+
+```console
+atr -d SPY
+rsi -d SPY
```
-
+There are now two new columns in the SPY dataset, `true_range, RSI_10_close`. To keep the dataset organized, it may be worth renaming these columns.
+```console
+rename -d SPY --oldcol RSI_10_close --newcol spy_rsi10
+rename -d SPY --oldcol true_range --newcol spy_atr
```
-(🦋) /forecast/ $ desc AAPL
- Showing Descriptive Statistics for Dataset AAPL
-┏━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
-┃ ┃ open ┃ high ┃ low ┃ close ┃ adj_close ┃ volume ┃
-┡━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
-│ count │ 758.00 │ 758.00 │ 758.00 │ 758.00 │ 758.00 │ 758.00 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ mean │ 121.57 │ 123.07 │ 120.15 │ 121.68 │ 120.72 │ 113864153.69 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ std │ 36.37 │ 36.74 │ 35.95 │ 36.34 │ 36.61 │ 56098731.72 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ min │ 53.47 │ 54.11 │ 52.77 │ 53.54 │ 52.38 │ 41000000.00 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ 25% │ 83.10 │ 86.41 │ 83.04 │ 84.16 │ 83.00 │ 77032650.00 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ 50% │ 128.18 │ 129.64 │ 126.63 │ 127.86 │ 126.70 │ 98135650.00 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ 75% │ 149.05 │ 150.38 │ 147.69 │ 149.14 │ 148.50 │ 131152875.00 │
-├───────┼────────┼────────┼────────┼────────┼───────────┼──────────────┤
-│ max │ 182.63 │ 182.94 │ 179.12 │ 182.01 │ 181.26 │ 426510000.00 │
-└───────┴────────┴────────┴────────┴────────┴───────────┴──────────────┘
+### Time Series Forecasting
+
+This group of features applies models to a target dataset and its columns. There are a wide selection to choose from. Please note that this guide is meant explain how to use the functions and does not attempt to explain the models themselves. This menu is an implementation of the [Unit8 Darts Time Series for Python](https://unit8.com/resources/darts-time-series-made-easy-in-python/) library.
+
+|Function |Description |Accepts Past Covariates? |
+|:--------|:------------|:-----------------------|
+|autoselect |Select best statistical model from AutoARIMA, AutoETS, AutoCES, MSTL, etc. |No |
+|autoarima |Automatic ARIMA Model |No |
+|autoces |Automatic Complex Exponential Smoothing Model |No |
+|autoets |Automatic ETS (Error, Trend, Seasonality) Model |No |
+|mstl |Multiple Seasonalities and Trend using Loess (MSTL) Model |No |
+|rwd |Random Walk with Drift Model |No |
+|seasonalnaive |Seasonal Naive Model |No |
+|expo |Probabilistic Exponential Smoothing |No |
+|theta |Theta Method |No |
+|linregr |Probabilistic Linear Regression |Yes |
+|regr |Regression |Yes |
+|brnn |Block Recurrent Neural Network (RNN, LSTM, GRU) |Yes |
+|nbeats |Neural Bayesian Estimation |Yes |
+|nhits |Neural Hierarchical Interpolation |Yes |
+|tcn |Temporal Convolutional Neural Network |Yes |
+|trans |Transformer Network |Yes |
+|tft |Temporal Fusion Transformer Network |Yes |
+
+Within this list of models, there are two distinct categories:
+
+- Without past covariates.
+- With past covariates.
+
+The models featuring past covariates will accept an unlimited number of columns, or all columns can be chosen. The latter being more appropriate for when a dataset contains only columns which are deemed fit for the purpose. To use any model in its default state, all that is required is the command name and the target dataset's name. The default target column will always be `close` so, a column must be defined to run a model if the target dataset does not contain a column with this name. With the target column as `close`, the basic default syntax will look like:
+
+```console
+rwd -d SPY
```
-Let's use a simple **Probabilistic Exponential Smoothing Model** to predict the close price. Keep in mind all models are perform automatic historical backtesting before providing future forecasts. Note that all models forecast `close` by default.
+
+Each model should be reviewed carefully to understand what the adjustable parameters are, and how they should be defined.
+
+### Quantile Anomaly Detection
+
+`anom` performs a Quantile Anomaly detection on a given dataset. Read more about this calculation [here](https://unit8co.github.io/darts/generated_api/darts.ad.detectors.quantile_detector.html).
+
+```console
+anom -d SPY
```
-(🦋) /forecast/ $ expo AAPL
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:17<00:00, 6.44it/s]
-Exponential smoothing obtains MAPE: 3.86%
+
+
+### Miscellaneous AI Tools
+
+#### Whisper
+
+The `whisper` feature allows users to transcribe, translate, and summarize videos on YouTube. These abilities empowers users to perform deeper research than ever before, and opens the door to a more complete view of the macroeconomic landscape. The models are not installed until the first time it is used, and they can be quite significant in size. Performance will vary, and there is currently not a method for offloading processing to dedicated GPUs.
+
+```console
+whisper https://www.youtube.com/watch?v=G0Q0BtGQzrA
- Actual price: 157.37
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-12 │ 157.99 │
-├────────────┼────────────┤
-│ 2022-09-13 │ 157.89 │
-├────────────┼────────────┤
-│ 2022-09-14 │ 158.32 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 158.25 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 158.52 │
-└────────────┴────────────┘
+[DISCLAIMER]: This is a beta feature that uses standard NLP models. More recent models such as GPT will be added in future releases.
+
+Downloading and Loading NLP Pipelines from cache...
+
+All NLP Pipelines loaded.
+
+Transcribing and summarizing...
+Downloaded video "China stuck in 8 trillion dollar debt crisis | GyanJaraHatke with Sourabh Maheshwari". Generating subtitles...
+Detected language: Hindi
+100%|████████████████████████████████████████████████████████| 83567/83567 [04:25<00:00, 314.21frames/s]
+100%|███████████████████████████████████████████████████████████████████| 11/11 [02:44<00:00, 14.99s/it]
+
+-------------------------
+Summary: Reduction: 81.38%
+Sentiment: NEGATIVE: 73.2596
+-------------------------
+China is the largest economy in the world. Usually China's economy is seen in the global level of the economy of America...
```
-
+A file with the transcript is saved to the OpenBBUserData folder.
+
+## Sample Workflow #1 (Beginner)
+
+Let's begin by using one of the datasets we loaded in previously : `SPY`
+
+We will be forecasting `5 Business days` ahead for the remaider of these workflows unless specified.
+
+**Note:** All models automatically perform Historical backtesting on the test split before providing a prediction.
-That looks great, but we might want to see it a little more up close. Lets set the flag for `--forecast-only`.
+We use [MAPE](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error) for the default as it is quite convenient and scale independent since it calculates error as a percentage instead of an absolute value. There are many more metrics to compare time series. The metrics will compare only common slices of series when the two series are not aligned, and parallelize computation over a large number of pairs of series. Additional metrics to choose from are RMSE, MSE, and SMAPE.
-
+### plot
-We can also play with some models that are bit more advanced. As we go down the list, models begin to become larger in parameter size and complexity. This will play a key role later on when we want to train models with `past_covariates` (aka. external factors).
+```console
+plot SPY.close
+```
-This time lets test with a **Recurrent Neural Network** which by default uses an `LSTM` backbone. We can also choose to test out a `GRU` backbone to experiment. Let's do both and see if we can improve our accuracy and reduce the overall MAPE.
+
+### desc
+
+```console
+desc SPY
```
-(🦋) /forecast/ $ rnn AAPL --forecast-only
-Epoch 193: 100%|███████████████████████████████████████████████████████████| 25/25 [00:00<00:00, 129.49it/s, loss=-2.74, train_loss=-2.75, val_loss=-2.22]
-Predicting RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:01<00:00, 86.89it/s]
-RNN model obtains MAPE: 3.69%
+
+
+### expo
+
+Let's use a simple **Probabilistic Exponential Smoothing Model** to predict the close price. Keep in mind all models are performing automatic historical backtesting before providing future forecasts. Note that all models forecast `close` by default.
- Actual price: 157.37
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-12 │ 155.71 │
-├────────────┼────────────┤
-│ 2022-09-13 │ 155.42 │
-├────────────┼────────────┤
-│ 2022-09-14 │ 155.11 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 154.81 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 154.51 │
-└────────────┴────────────┘
+```console
+expo SPY
```
-
+
-Seems like we improved the performance and reduced MAPE!
+### rnn
-Now for the second task, we would like to change the model type from `LSTM` --> `GRU`. Use the `-h` flag to understand the particular parameters one can change for RNN. (Please note that the parameters are different for each model).
+We can also play with some models that are bit more advanced. As we go down the list, models begin to become larger in parameter size and complexity. This will play a key role later on when we want to train models with `past_covariates` (aka. external factors).
+This time lets test with a **Recurrent Neural Network** which by default uses an `LSTM` backbone. We can also choose to test out a `GRU` backbone to experiment. Let's do both and see if we can improve our accuracy and reduce the overall MAPE.
+
+```console
+rnn SPY --forecast-only
+
+Epoch 50: 100%|█████████████████████████████████████| 214/214 [00:02<00:00, 77.77it/s, loss=-4.31, v_num=logs, train_loss=-4.46, val_loss=-.985]
+
+RNN model obtains MAPE: 3.13%
```
-(🦋) /forecast/ $ rnn -h
-usage: rnn [--hidden-dim HIDDEN_DIM] [--training_length TRAINING_LENGTH] [--naive] [-d {AAPL,msft}] [-c TARGET_COLUMN] [-n N_DAYS] [-t TRAIN_SPLIT]
- [-i INPUT_CHUNK_LENGTH] [--force-reset FORCE_RESET] [--save-checkpoints SAVE_CHECKPOINTS] [--model-save-name MODEL_SAVE_NAME]
- [--n-epochs N_EPOCHS] [--model-type MODEL_TYPE] [--dropout DROPOUT] [--batch-size BATCH_SIZE] [--end S_END_DATE] [--start S_START_DATE]
- [--learning-rate LEARNING_RATE] [--residuals] [--forecast-only] [-h] [--export EXPORT]
-Perform RNN forecast (Vanilla RNN, LSTM, GRU)
+
+
+This result expresses a different view from the Probabilistic Exponential Smoothing Model.
+
+For the second task, we would like to change the model type from `LSTM` --> `GRU`. Let's find out of this improves the MAPE score. Use the `-h` flag to understand the particular parameters one can change for RNN. (Please note that the parameters are different for each model).
+
+```console
+rnn -h
+
+usage: rnn [--hidden-dim HIDDEN_DIM] [--training_length TRAINING_LENGTH] [--naive] [-d {vix,SPY}] [-c TARGET_COLUMN] [-n N_DAYS]
+ [-t TRAIN_SPLIT] [-i INPUT_CHUNK_LENGTH] [--force-reset FORCE_RESET] [--save-checkpoints SAVE_CHECKPOINTS]
+ [--model-save-name MODEL_SAVE_NAME] [--n-epochs N_EPOCHS] [--model-type MODEL_TYPE] [--dropout DROPOUT] [--batch-size BATCH_SIZE]
+ [--end S_END_DATE] [--start S_START_DATE] [--learning-rate LEARNING_RATE] [--residuals] [--forecast-only] [--export-pred-raw]
+ [--metric {rmse,mse,mape,smape}] [-h] [--export EXPORT]
-optional arguments:
+Perform RNN forecast (Vanilla RNN, LSTM, GRU): https://unit8co.github.io/darts/generated_api/darts.models.forecasting.rnn_model.html
+
+options:
--hidden-dim HIDDEN_DIM
Size for feature maps for each hidden RNN layer (h_n) (default: 20)
--training_length TRAINING_LENGTH
- The length of both input (target and covariates) and output (target) time series used during training. Generally speaking,
- training_length should have a higher value than input_chunk_length because otherwise during training the RNN is never run for as
- many iterations as it will during training. (default: 20)
+ The length of both input (target and covariates) and output (target) time series used during training. Generally
+ speaking, training_length should have a higher value than input_chunk_length because otherwise during training the RNN
+ is never run for as many iterations as it will during training. (default: 20)
--naive Show the naive baseline for a model. (default: False)
- -d {AAPL,msft}, --target-dataset {AAPL,msft}
+ -d {vix,SPY}, --dataset {vix,SPY}
The name of the dataset you want to select (default: None)
-c TARGET_COLUMN, --target-column TARGET_COLUMN
The name of the specific column you want to use (default: close)
@@ -205,16 +365,16 @@ optional arguments:
-i INPUT_CHUNK_LENGTH, --input-chunk-length INPUT_CHUNK_LENGTH
Number of past time steps for forecasting module at prediction time. (default: 14)
--force-reset FORCE_RESET
- If set to True, any previously-existing model with the same name will be reset (all checkpoints will be discarded). (default:
- True)
+ If set to True, any previously-existing model with the same name will be reset (all checkpoints will be discarded).
+ (default: True)
--save-checkpoints SAVE_CHECKPOINTS
Whether to automatically save the untrained model and checkpoints. (default: True)
--model-save-name MODEL_SAVE_NAME
Name of the model to save. (default: rnn_model)
--n-epochs N_EPOCHS Number of epochs over which to train the model. (default: 300)
--model-type MODEL_TYPE
- Either a string specifying the RNN module type ("RNN", "LSTM" or "GRU") (default: LSTM)
- --dropout DROPOUT Fraction of neurons afected by Dropout. (default: 0)
+ Enter a string specifying the RNN module type ("RNN", "LSTM" or "GRU") (default: LSTM)
+ --dropout DROPOUT Fraction of neurons affected by Dropout, from 0 to 1. (default: 0)
--batch-size BATCH_SIZE
Number of time series (input and output) used in each training pass (default: 32)
--end S_END_DATE The end date (format YYYY-MM-DD) to select for testing (default: None)
@@ -222,46 +382,37 @@ optional arguments:
--learning-rate LEARNING_RATE
Learning rate during training. (default: 0.001)
--residuals Show the residuals for the model. (default: False)
- --forecast-only Do not plot the hisotorical data without forecasts. (default: False)
+ --forecast-only Do not plot the historical data without forecasts. (default: False)
+ --export-pred-raw Export predictions to a csv file. (default: False)
+ --metric {rmse,mse,mape,smape}
+ Calculate precision based on a specific metric (rmse, mse, mape) (default: mape)
-h, --help show this help message (default: False)
--export EXPORT Export figure into png, jpg, pdf, svg (default: )
+
+For more information and examples, use 'about rnn' to access the related guide.
```
Lets change the `--model-type` parameter to `GRU` and rerun.
-```
-(🦋) /forecast/ $ rnn AAPL --model-type GRU --forecast-only
-Epoch 35: 100%|████████████████████████████████████████████████████████████| 25/25 [00:00<00:00, 125.85it/s, loss=-2.72, train_loss=-2.74, val_loss=-2.13]
-Predicting RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:01<00:00, 85.14it/s]
-RNN model obtains MAPE: 3.64%
+```console
+rnn SPY --model-type GRU --forecast-only
+Epoch 85: 100%|█████████████████████████████████████| 214/214 [00:02<00:00, 75.10it/s, loss=-4.45, v_num=logs, train_loss=-4.35, val_loss=-2.35]
- Actual price: 157.37
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-12 │ 155.28 │
-├────────────┼────────────┤
-│ 2022-09-13 │ 155.02 │
-├────────────┼────────────┤
-│ 2022-09-14 │ 154.65 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 154.30 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 153.96 │
-└────────────┴────────────┘
+RNN model obtains MAPE: 2.77%
```
-
+
-Looks like we squeezed out a little bit more accuracy! Good work.
+We improved the accuracy score, great work!
-The take away for this is that all models should work out of the box when forecasting for a particular Timeseries. One can switch the target by specifying a `-c` for `TARGET_COLUMN` and test out performance with multiple different models with a single command.
+:::note
+The take away for this is that all models should work out of the box when forecasting for a particular time series. One can switch the target by specifying a `-c` for `TARGET_COLUMN` and test out performance with multiple different models with a single command.
+:::
-### Sample workflow #2 (advanced)
+## Sample Workflow #2 (Advanced)
-To build successful models and improve accuracy over time, it is important to capture external data related to the timeseries you are training on. This can be seen in everyday applications:
+To build successful models and improve accuracy over time, it is important to capture external data related to the time series you are training on. This can be seen in everyday applications:
- Observed rainfalls and known weather forecasts can help to predict hydro and solar electricity production
- Recently-observed activity on an e-commerce website can help predict future sales.
@@ -275,302 +426,220 @@ time. Those series often contain values that have to be observed to be known.
-If you would like to explore this topic more, please read the [blog post](https://medium.com/unit8-machine-learning-publication/time-series-forecasting-using-past-and-future-external-data-with-darts-1f0539585993) written by the authors of Darts.
+To explore this topic more, please read the [blog post](https://medium.com/unit8-machine-learning-publication/time-series-forecasting-using-past-and-future-external-data-with-darts-1f0539585993) written by the authors of Darts.
Note that only the following models can handle `past_covariates`: `BlockRNNModel`, `NBEATSModel`, `TCNModel`, `TransformerModel`, `RegressionModel` (incl. `LinearRegressionModel`), `Temporal Fusion Transformer`
-In this work flow lets do the following:
+Earlier on in the guide we managed to accomplish the following:
-- add in some correlation analysis
-- combine datasets
-- perform feature engineering
-- train models with `past_covariates`
+- Load multiple datasets.
+- Combine datasets.
+- Rename columns.
+- Delete columns.
-Let's begin by loading in our datasets of AAPL and MSFT once again. In this work flow we are going to test if `close` price of MSFT is at all affected by the `close` of AAPL.
+We will continue this workflow by:
-For a refresher, we will grab data from the `stocks` menu found on the terminal.
+- Adding some correlation analysis.
+- Train models with `past_covariates`.
-```
-(🦋) / $ stocks
-(🦋) /stocks/ $ load AAPL
-(🦋) /stocks/ $ forecast
-(🦋) /forecast/ $ ..
-(🦋) /stocks/ $ load MSFT
-(🦋) /stocks/ $ forecast
+For practice, let's start fresh and rebuild the same dataset. The [OpenBB Routine Scripts](https://docs.openbb.co/terminal/usage/guides/scripts-and-routines) can make quick work out of this chore. Copy the block below and create a new `.openbb` file, in `~/OpenBBUserData/routines/`, to follow along.
+
+```console
+/stocks
+load SPY --start 1997-01-01
+forecast
+..
+load ^VIX --start 1997-01-01
+forecast
+combine --dataset SPY -c ^VIX.high
+combine --dataset SPY -c ^VIX.low
+delete --delete SPY.adj_close
+delete --delete SPY.stock_splits
+delete --delete SPY.dividends
+atr -d SPY
+rsi -d SPY
+rename -d SPY --oldcol RSI_10_close --newcol SPY_rsi10
+rename -d SPY --oldcol true_range --newcol SPY_atr
+show SPY
```
-
+Restart the Terminal to ensure that the routine file is found by the `/exe` command, and then run it.
-Before we go combining them, let's train a simple `Block RNN` model on MSFT `close` price to see how to use `past_covariates`
+```console
+exe --file forecast_demo.openbb
+```
-Make sure to always check your current data set to know the column names:
+
-```
-(🦋) /forecast/ $ show MSFT
-MSFT dataset has shape (row, column): (759, 7)
+### brnn
- Dataset MSFT | Showing 10 of 759 rows
-┏━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┓
-┃ ┃ date ┃ open ┃ high ┃ low ┃ close ┃ adj_close ┃ volume ┃
-┡━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━┩
-│ 0 │ 2019-09-10 00:00:00 │ 136.80 │ 136.89 │ 134.51 │ 136.08 │ 132.22 │ 28903400 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 1 │ 2019-09-11 00:00:00 │ 135.91 │ 136.27 │ 135.09 │ 136.12 │ 132.26 │ 24726100 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 2 │ 2019-09-12 00:00:00 │ 137.85 │ 138.42 │ 136.87 │ 137.52 │ 133.62 │ 27010000 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 3 │ 2019-09-13 00:00:00 │ 137.78 │ 138.06 │ 136.57 │ 137.32 │ 133.42 │ 23363100 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 4 │ 2019-09-16 00:00:00 │ 135.83 │ 136.70 │ 135.66 │ 136.33 │ 132.46 │ 16731400 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 5 │ 2019-09-17 00:00:00 │ 136.96 │ 137.52 │ 136.43 │ 137.39 │ 133.49 │ 17814200 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 6 │ 2019-09-18 00:00:00 │ 137.36 │ 138.67 │ 136.53 │ 138.52 │ 134.59 │ 23982100 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 7 │ 2019-09-19 00:00:00 │ 140.30 │ 142.37 │ 140.07 │ 141.07 │ 137.07 │ 35772100 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 8 │ 2019-09-20 00:00:00 │ 141.01 │ 141.65 │ 138.25 │ 139.44 │ 135.48 │ 39167300 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┤
-│ 9 │ 2019-09-23 00:00:00 │ 139.23 │ 139.63 │ 138.44 │ 139.14 │ 135.19 │ 17139300 │
-└───┴─────────────────────┴────────┴────────┴────────┴────────┴───────────┴──────────┘
-```
+Now take the new dataset and train a simple `Block RNN` model on SPY's `close` price, and again using `past_covariates` on a single column.
-Without any covariates:
+Without past covariates:
+```console
+brnn SPY --forecast-only
```
-(🦋) /forecast/ $ brnn MSFT --forecast-only
-Epoch 87: 100%|█████████████████████████████████████████████████████████████| 25/25 [00:00<00:00, 33.84it/s, loss=-2.06, train_loss=-2.27, val_loss=-1.82]
-Predicting Block RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:06<00:00, 18.76it/s]
-Block RNN model obtains MAPE: 4.62%
+```console
+Epoch 196: 100%|███████████████████████████████████| 214/214 [00:01<00:00, 107.42it/s, loss=-4.09, v_num=logs, train_loss=-4.07, val_loss=-1.53]
- Actual price: 251.99
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-14 │ 258.52 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 258.01 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 258.95 │
-├────────────┼────────────┤
-│ 2022-09-19 │ 257.20 │
-├────────────┼────────────┤
-│ 2022-09-20 │ 257.66 │
-└────────────┴────────────┘
+Block RNN model obtains MAPE: 3.25%
```
-
+
With covariates:
-To use any covariates, you have 2 options:
+To use any covariates, there are two options:
- specify specific columns with `--past-covariates`
- specify all columns as past covariates except the one you are forecasting
`--all-past-covariates`
+```console
+brnn SPY --forecast-only --past-covariates volume
```
-(🦋) /forecast/ $ brnn MSFT --forecast-only --past-covariates volume
+
+```console
Covariate #0: volume
-Epoch 37: 100%|████████████████████████████████████████████████████████████| 25/25 [00:00<00:00, 149.03it/s, loss=-2.16, train_loss=-2.08, val_loss=-1.44]
-Predicting Block RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:01<00:00, 88.94it/s]
-Block RNN model obtains MAPE: 5.10%
+Epoch 86: 100%|████████████████████████████████████████| 214/214 [00:02<00:00, 74.77it/s, loss=-4, v_num=logs, train_loss=-4.19, val_loss=-1.53]
+Block RNN model obtains MAPE: 4.69%
+```
+
+
+It is evident here that adding in the external variable of `volume` negatively affected the accuracy.
+
+Let's try adding a new column for the 200-day moving average.
+
+```console
+ema -d SPY --period 200/brnn -d SPY --past-covariates EMA_200
+```
- Actual price: 251.99
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-14 │ 257.02 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 255.82 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 260.13 │
-├────────────┼────────────┤
-│ 2022-09-19 │ 258.32 │
-├────────────┼────────────┤
-│ 2022-09-20 │ 258.52 │
-└────────────┴────────────┘
+```console
+Successfully added 'EMA_200' to 'SPY' dataset
+Covariate #0: EMA_200
+Epoch 79: 100%|█████████████████████████████████████| 214/214 [00:02<00:00, 87.78it/s, loss=-3.81, v_num=logs, train_loss=-3.78, val_loss=-.435
+Block RNN model obtains MAPE: 4.60%
```
-
+
-You can see here that adding in the external variable of `volume` negatively affected the accuracy.
+This isn't an improvement. It's possible that there is too much noise in the data, so let's try shortening the length of time being used to train.
-Let's add in all remaining columns from our dataset as covariates and see what happens to the accuracy.
+```console
+brnn -d SPY --past-covariates EMA_200 -t 0.95
+
+Covariate #0: EMA_200
+Epoch 87: 100%|█████████████████████████████████████| 215/215 [00:02<00:00, 82.81it/s, loss=-3.87, v_num=logs, train_loss=-4.29, val_loss=-1.59]
+Block RNN model obtains MAPE: 3.38%
```
-(🦋) /forecast/ $ brnn MSFT --forecast-only --all-past-covariates
+
+
+
+Removing the 2020 volatility from the window of observation made a massive improvement to the forecast. Now let's see add all the columns in the dataset as past covariates.
+
+```console
+brnn -d SPY --all-past-covariates -t 0.95
+
Covariate #0: open
Covariate #1: high
Covariate #2: low
-Covariate #3: adj_close
-Covariate #4: volume
-Epoch 50: 100%|████████████████████████████████████████████████████████████| 25/25 [00:00<00:00, 149.29it/s, loss=-2.41, train_loss=-2.53, val_loss=-1.71]
-Predicting Block RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:01<00:00, 89.43it/s]
-Block RNN model obtains MAPE: 4.26%
-
-
- Actual price: 251.99
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-14 │ 259.28 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 259.13 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 259.42 │
-├────────────┼────────────┤
-│ 2022-09-19 │ 257.73 │
-├────────────┼────────────┤
-│ 2022-09-20 │ 258.64 │
-└────────────┴────────────┘
-```
-
-
-
-**Final Result:** Using `open`, `high`, `low`, `adj_close`, `volume` as
-`past_covariates` improved MAPE from 4.62 --> 4.26.
-
-Now that we know how to use covariates and understand their effect, why don't we also use `AAPL`'s ticker data as `past_covariates` to check whether this correlates and improves our forecasting accuracy.
-
-**Remember: You can use unlimited number of `past_covariates` but they must all be combined into a single dataframe with the target forecast timeseries before training.**
-
-We will combine `MSFT` and `AAPL`.
-
-```
-(🦋) /forecast/ $ combine MSFT -c AAPL
-(🦋) /forecast/ $ show MSFT
-MSFT dataset has shape (row, column): (759, 13)
-Dataframe has more than 10 columns. Please export to see all of the data.
-
-
- Dataset MSFT | Showing 10 of 759 rows
-┏━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┓
-┃ ┃ date ┃ open ┃ high ┃ low ┃ close ┃ adj_close ┃ volume ┃ AAPL_open ┃ AAPL_high ┃ AAPL_low ┃
-┡━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━┩
-│ 0 │ 2019-09-10 00:00:00 │ 136.80 │ 136.89 │ 134.51 │ 136.08 │ 132.22 │ 28903400 │ 53.47 │ 54.19 │ 52.93 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 1 │ 2019-09-11 00:00:00 │ 135.91 │ 136.27 │ 135.09 │ 136.12 │ 132.26 │ 24726100 │ 54.52 │ 55.93 │ 54.43 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 2 │ 2019-09-12 00:00:00 │ 137.85 │ 138.42 │ 136.87 │ 137.52 │ 133.62 │ 27010000 │ 56.20 │ 56.60 │ 55.72 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 3 │ 2019-09-13 00:00:00 │ 137.78 │ 138.06 │ 136.57 │ 137.32 │ 133.42 │ 23363100 │ 55.00 │ 55.20 │ 54.26 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 4 │ 2019-09-16 00:00:00 │ 135.83 │ 136.70 │ 135.66 │ 136.33 │ 132.46 │ 16731400 │ 54.43 │ 55.03 │ 54.39 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 5 │ 2019-09-17 00:00:00 │ 136.96 │ 137.52 │ 136.43 │ 137.39 │ 133.49 │ 17814200 │ 54.99 │ 55.21 │ 54.78 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 6 │ 2019-09-18 00:00:00 │ 137.36 │ 138.67 │ 136.53 │ 138.52 │ 134.59 │ 23982100 │ 55.26 │ 55.71 │ 54.86 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 7 │ 2019-09-19 00:00:00 │ 140.30 │ 142.37 │ 140.07 │ 141.07 │ 137.07 │ 35772100 │ 55.50 │ 55.94 │ 55.09 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 8 │ 2019-09-20 00:00:00 │ 141.01 │ 141.65 │ 138.25 │ 139.44 │ 135.48 │ 39167300 │ 55.35 │ 55.64 │ 54.37 │
-├───┼─────────────────────┼────────┼────────┼────────┼────────┼───────────┼──────────┼───────────┼───────────┼──────────┤
-│ 9 │ 2019-09-23 00:00:00 │ 139.23 │ 139.63 │ 138.44 │ 139.14 │ 135.19 │ 17139300 │ 54.74 │ 54.96 │ 54.41 │
-└───┴─────────────────────┴────────┴────────┴────────┴────────┴───────────┴──────────┴───────────┴───────────┴──────────┘
-```
-
-Now we can run the same `BRNN` model with all `past_covariates` of both tickers. The output will show the model grabbing all covariates to bring into training to predict `close` of `MSFT`.
-
-```
-(🦋) /forecast/ $ brnn MSFT --forecast-only --all-past-covariates
+Covariate #3: volume
+Covariate #4: ^VIX_high
+Covariate #5: ^VIX_low
+Covariate #6: SPY_atr
+Covariate #7: SPY_rsi10
+Covariate #8: EMA_200
+Epoch 39: 100%|█████████████████████████████████████| 215/215 [00:02<00:00, 83.50it/s, loss=-3.92, v_num=logs, train_loss=-4.03, val_loss=-2.14]
+Block RNN model obtains MAPE: 3.57%
+```
+
+
+
+Adding all of the columns that were created doesn't improve the MAPE score, but it does narrow the range of forecasted prices. Playing around like this shows how small changes can have a large impact on a forecast. It is important to test many variables and parameters without getting too caught up overfitting any particular model. Validate a thesis before dedicating a large amount of time into it.
+
+### regr
+
+Using the `regr` function with `all-past-covariates` on the same dataset gives dramatically different forecast.
+
+```console
+regr SPY --all-past-covariates
+
Covariate #0: open
Covariate #1: high
Covariate #2: low
-Covariate #3: adj_close
-Covariate #4: volume
-Covariate #5: AAPL_open
-Covariate #6: AAPL_high
-Covariate #7: AAPL_low
-Covariate #8: AAPL_close
-Covariate #9: AAPL_adj_close
-Covariate #10: AAPL_volume
-Epoch 116: 100%|████████████████████████████████████████████████████████████| 25/25 [00:00<00:00, 149.26it/s, loss=-2.5, train_loss=-2.61, val_loss=-1.93]
-Predicting Block RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:01<00:00, 90.26it/s]
-Block RNN model obtains MAPE: 3.93%
+Covariate #3: volume
+Covariate #4: ^VIX_high
+Covariate #5: ^VIX_low
+Covariate #6: SPY_atr
+Covariate #7: SPY_rsi10
+Predicting Regression for 5 days
+100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 1026/1026 [00:48<00:00, 21.35it/s]
+Regression model obtains MAPE: 2.68%
+```
+
+
+Now that we know how to use covariates, and are starting to understand their effects, let's examine the impact of MSFT and AAPL closing prices on the regression forecast for SPY. We will discard the previous work and start fresh.
- Actual price: 251.99
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-14 │ 259.78 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 258.15 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 261.44 │
-├────────────┼────────────┤
-│ 2022-09-19 │ 257.57 │
-├────────────┼────────────┤
-│ 2022-09-20 │ 257.99 │
-└────────────┴────────────┘
+The cache can be purged by resetting the Terminal. Use this command to clear it:
+
+```console
+/r
```
-
+Now we will fetch the data and combine the columns:
-For one last experiment, we can perform some other feature engineering on `MSFT` and add it to our `past_covariates` to train on.
+```console
+/stocks/load spy
+forecast
+..
+load aapl
+forecast
+..
+load msft
+forecast
+delete --delete SPY.stock_splits
+delete --delete SPY.adj_close
+delete --delete SPY.dividends
+combine SPY -c AAPL.close
+combine SPY -c MSFT.close
+regr SPY --forecast-only --past-covariates AAPL_close,MSFT_close
+```
-In this case, let's add in `Momentum` over past 10 days of `MSFT` and append it to our `past_covariates`
+**Remember: You can use unlimited number of `past_covariates` but they must all be combined into a single dataframe with the target forecast time series before training.**
+```console
+Covariate #0: AAPL_close
+Covariate #1: MSFT_close
+Predicting Regression for 5 days
+100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:01<00:00, 96.07it/s]
+Regression model obtains MAPE: 1.77%
```
-(🦋) /forecast/ $ mom MSFT
-Successfully added 'Momentum_10' to 'MSFT' dataset
-(🦋) /forecast/ $ brnn MSFT --forecast-only --all-past-covariates
-The data contains inf or nan values. They will be removed.
+
+
+Adding the rest of the columns as past covariates improves the MAPE score slightly and starts to take a more directional view.
+
+```console
+regr --all-past-covariates -d SPY
Covariate #0: open
Covariate #1: high
Covariate #2: low
-Covariate #3: adj_close
-Covariate #4: volume
-Covariate #5: AAPL_open
-Covariate #6: AAPL_high
-Covariate #7: AAPL_low
-Covariate #8: AAPL_close
-Covariate #9: AAPL_adj_close
-Covariate #10: AAPL_volume
-Covariate #11: Momentum_10
-Epoch 71: 100%|████████████████████████████████████████████████████████████| 24/24 [00:00<00:00, 147.87it/s, loss=-2.52, train_loss=-2.57, val_loss=-1.82]
-Predicting Block RNN for 5 days
-100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 114/114 [00:01<00:00, 90.05it/s]
-Block RNN model obtains MAPE: 3.72%
-
-
- Actual price: 251.99
-┏━━━━━━━━━━━━┳━━━━━━━━━━━━┓
-┃ Datetime ┃ Prediction ┃
-┡━━━━━━━━━━━━╇━━━━━━━━━━━━┩
-│ 2022-09-14 │ 255.14 │
-├────────────┼────────────┤
-│ 2022-09-15 │ 256.20 │
-├────────────┼────────────┤
-│ 2022-09-16 │ 256.67 │
-├────────────┼────────────┤
-│ 2022-09-19 │ 258.63 │
-├────────────┼────────────┤
-│ 2022-09-20 │ 256.62 │
-└────────────┴────────────┘
-```
-
-
-
-There we have it. Bringing in another ticker has allowed us to further improve the model accuracy. Furthermore, adding in a new feature to the dataset allowed us to improve the accuracy further.
-
-**MAPE = 4.62%** (no past covariates)
-
-**MAPE = 4.26%** (`open`,`high`,`low`,`adj_close`,`volume`)
-
-**MAPE = 3.93%**
-(`open`,`high`,`low`,`adj_close`,`volume`,`AAPL_open`,`AAPL_high`,`APPL_low`,`APPL_adj_close`,`APPL_volume`,`APPL_close`)
-
-**MAPE = 3.72%**
-(`open`,`high`,`low`,`adj_close`,`volume`,`AAPL_open`,`AAPL_high`,`APPL_low`,`APPL_adj_close`,`APPL_volume`,`APPL_close`,`Momentum_10`)
-
-If you have any questions or would like to request for new feature engineering or model additions, please join us on [Discord](openbb.co/discord). Happy hacking!
+Covariate #3: volume
+Covariate #4: AAPL_close
+Covariate #5: MSFT_close
+Predicting Regression for 5 days
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:00<00:00, 117.16it/s]
+Regression model obtains MAPE: 1.66%
+```
+
+
+
+:::note
+The examples here are over-simplified as a means for demonstrating a framework to create and conduct experiments. It is important to keep in mind that these are tools, they are not oracles, and that results may vary. If you have any questions, requests for new feature engineering and model additions, or just want to be part of the conversation, please join us on [Discord](openbb.co/discord). Happy hacking!
+:::