[Feature] Automatic Video Analysis with NLP 📺

.github/workflows/intel_macos_build.yml

@@ -52,7 +52,7 @@ jobs:
           environment-file: build/conda/conda-3-9-env-full.yaml
           activate-environment: build_env
           use-only-tar-bz2: true # Needed for caching some reason
-      - name: Run Poetry
+      - name: Install requirements
         run: |
           pip list
           python -m pip install -r requirements-full.txt

.gitignore

@@ -62,5 +62,9 @@ darts_logs/
 custom_imports/*.csv
 custom_imports/*/*.csv
 
+# cache
+cache/
+
 # lightning logs
 lightning_logs/
+

build/pyinstaller/hooks/hook-whisper.py

@@ -0,0 +1,35 @@
+import importlib
+
+import importlib_metadata
+from PyInstaller.utils.hooks import copy_metadata
+
+datas = copy_metadata("transformers")
+datas += copy_metadata("tokenizers")
+datas += copy_metadata("tqdm")
+datas += copy_metadata("regex")
+datas += copy_metadata("requests")
+datas += copy_metadata("packaging")
+datas += copy_metadata("filelock")
+datas += copy_metadata("numpy")
+datas += copy_metadata("torch")
+
+candidates = [
+    "tensorflow",
+    "tensorflow-cpu",
+    "tensorflow-gpu",
+    "tf-nightly",
+    "tf-nightly-cpu",
+    "tf-nightly-gpu",
+    "intel-tensorflow",
+    "intel-tensorflow-avx512",
+    "tensorflow-rocm",
+    "tensorflow-macos",
+    "tensorflow-aarch64",
+]
+for candidate in candidates:
+    try:
+        if importlib.util.find_spec(candidate):
+            datas += copy_metadata(candidate)
+            break
+    except importlib_metadata.PackageNotFoundError:
+        pass

build/pyinstaller/terminal.spec

@@ -60,6 +60,8 @@ added_files = [
     (os.path.join(pathex, "user_agent"), "user_agent"),
     (os.path.join(pathex, "vaderSentiment"), "vaderSentiment"),
     (os.path.join(pathex, "prophet"), "prophet"),
+    (os.path.join(pathex, "whisper"), "whisper"),
+    (os.path.join(pathex, "transformers"), "transformers"),
     (
         os.path.join(pathex, "linearmodels", "datasets"),
         os.path.join("linearmodels", "datasets"),
@@ -77,10 +79,14 @@ added_files = [
     (os.path.join(pathex, "blib2to3", "PatternGrammar.txt"), "blib2to3"),
 ]
 if is_win:
-    added_files.append(
-        (os.path.join(f"{os.path.dirname(scipy.__file__)}.libs"), "scipy.libs/"),
+    added_files.extend(
+        [
+            (os.path.join(f"{os.path.dirname(scipy.__file__)}.libs"), "scipy.libs/"),
+            (os.path.join(pathex, "frozendict", "version.py"), "frozendict"),
+        ]
     )
 
+
 # Python libraries that are explicitly pulled into the bundle
 hidden_imports = [
     "sklearn.utils._cython_blas",
@@ -98,15 +104,21 @@ hidden_imports = [
     "statsmodels",
     "user_agent",
     "vaderSentiment",
-    "pyEX",
     "feedparser",
     "_sysconfigdata__darwin_darwin",
     "prophet",
     "debugpy",
     "scipy.sparse.linalg._isolve._iterative",
+    "whisper",
+    "transformers",
+    "yt_dlp",
+    "textwrap3",
 ]
 
 
+if is_win:
+    hidden_imports.append("frozendict")
+
 analysis_kwargs = dict(
     scripts=[os.path.join(os.getcwd(), "terminal.py")],
     pathex=[pathex, "."],

openbb_terminal/core/config/paths.py

@@ -32,3 +32,4 @@ def get_user_data_directory():
 USER_REPORTS_DIRECTORY = USER_DATA_DIRECTORY / "reports"
 USER_CUSTOM_REPORTS_DIRECTORY = USER_DATA_DIRECTORY / "reports" / "custom reports"
 USER_FORECAST_MODELS_DIRECTORY = USER_DATA_DIRECTORY / "exports" / "forecast_models"
+USER_FORECAST_WHISPER_DIRECTORY = USER_DATA_DIRECTORY / "exports" / "whisper"

openbb_terminal/core/sdk/models/economy_sdk_model.py

@@ -13,6 +13,8 @@ class EconomyRoot(Category):
         `bigmac`: Display Big Mac Index for given countries\n
         `bigmac_chart`: Display Big Mac Index for given countries\n
         `country_codes`: Get available country codes for Bigmac index\n
+        `cpi`: Obtain CPI data from FRED. [Source: FRED]\n
+        `cpi_chart`: Plot CPI data. [Source: FRED]\n
         `currencies`: Scrape data for global currencies\n
         `events`: Get economic calendar for countries between specified dates\n
         `fred`: Get Series data. [Source: FRED]\n
@@ -52,6 +54,8 @@ def __init__(self):
         self.bigmac = lib.economy_nasdaq_model.get_big_mac_indices
         self.bigmac_chart = lib.economy_nasdaq_view.display_big_mac_index
         self.country_codes = lib.economy_nasdaq_model.get_country_codes
+        self.cpi = lib.economy_fred_model.get_cpi
+        self.cpi_chart = lib.economy_fred_view.plot_cpi
         self.currencies = lib.economy_wsj_model.global_currencies
         self.events = lib.economy_nasdaq_model.get_economic_calendar
         self.fred = lib.economy_fred_model.get_aggregated_series_data

openbb_terminal/core/sdk/models/forecast_sdk_model.py

@@ -150,3 +150,4 @@ def __init__(self):
             self.theta_chart = lib.forecast_theta_view.display_theta_forecast
             self.trans = lib.forecast_trans_model.get_trans_data
             self.trans_chart = lib.forecast_trans_view.display_trans_forecast
+            self.whisper = lib.forecast_whisper_model.transcribe_and_summarize

openbb_terminal/core/sdk/sdk_init.py

@@ -465,6 +465,7 @@
         theta_view as forecast_theta_view,
         trans_model as forecast_trans_model,
         trans_view as forecast_trans_view,
+        whisper_model as forecast_whisper_model,
     )
 except ImportError:
     FORECASTING_TOOLKIT_ENABLED = False

openbb_terminal/core/sdk/trail_map.csv

@@ -181,6 +181,7 @@ econometrics.root,econometrics_model.get_root,econometrics_view.display_root
 economy.available_indices,economy_yfinance_model.get_available_indices,
 economy.bigmac,economy_nasdaq_model.get_big_mac_indices,economy_nasdaq_view.display_big_mac_index
 economy.country_codes,economy_nasdaq_model.get_country_codes,
+economy.cpi,economy_fred_model.get_cpi,economy_fred_view.plot_cpi
 economy.currencies,economy_wsj_model.global_currencies,
 economy.events,economy_nasdaq_model.get_economic_calendar,
 economy.fred,economy_fred_model.get_aggregated_series_data,economy_fred_view.display_fred_series
@@ -203,7 +204,6 @@ economy.search_index,economy_yfinance_model.get_search_indices,
 economy.spectrum,economy_finviz_view.display_spectrum,
 economy.treasury,economy_econdb_model.get_treasuries,economy_econdb_view.show_treasuries
 economy.treasury_maturities,economy_econdb_model.get_treasury_maturities,
-economy.cpi,economy_fred_model.get_cpi,economy_fred_view.plot_cpi,
 economy.usbonds,economy_wsj_model.us_bonds,
 economy.valuation,economy_finviz_model.get_valuation_data,
 etf.candle,stocks_helper.display_candle,

openbb_terminal/core/sdk/trail_map_forecasting.csv

@@ -37,3 +37,4 @@ forecast.tcn,forecast_tcn_model.get_tcn_data,forecast_tcn_view.display_tcn_forec
 forecast.tft,forecast_tft_model.get_tft_data,forecast_tft_view.display_tft_forecast
 forecast.theta,forecast_theta_model.get_theta_data,forecast_theta_view.display_theta_forecast
 forecast.trans,forecast_trans_model.get_trans_data,forecast_trans_view.display_trans_forecast
+forecast.whisper,forecast_whisper_model.transcribe_and_summarize,

openbb_terminal/forecast/forecast_controller.py

@@ -25,17 +25,61 @@
         "Please install the forecast version of the terminal. Instructions can be found "
         "under the python tab: https://docs.openbb.co/terminal/quickstart/installation"
     )
+
+try:
+    import whisper
+    import transformers
+    from whisper.tokenizer import LANGUAGES, TO_LANGUAGE_CODE
+    from openbb_terminal.forecast.whisper_utils import str2bool
+
+    transformers_ver = transformers.__version__
+    # if imports are successful, set flag to True
+    WHISPER_AVAILABLE = True
+
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "Please use poetry to install latest whisper model and dependencies. \n"
+        "poetry install -E forecast \n"
+        "\n"
+        "If you are not using poetry, please install whisper model. Instructions can be found here: \n"
+        "https://github.com/openai/whisper \n"
+        "Please install the transformers library with the following command: \n"
+        "pip install transformers \n"
+    )
+
 import pandas as pd
 import psutil
 
+
+# ignore  pylint(ungrouped-imports)
+# pylint: disable=ungrouped-imports
+
 from openbb_terminal import feature_flags as obbff
 from openbb_terminal.common import common_model
+
 from openbb_terminal.core.config.paths import (
     USER_CUSTOM_IMPORTS_DIRECTORY,
     USER_EXPORTS_DIRECTORY,
+    USER_FORECAST_WHISPER_DIRECTORY,
 )
 from openbb_terminal.custom_prompt_toolkit import NestedCompleter
 from openbb_terminal.decorators import log_start_end
+
+from openbb_terminal.helper_funcs import (
+    check_positive,
+    check_positive_float,
+    NO_EXPORT,
+    EXPORT_ONLY_FIGURES_ALLOWED,
+    EXPORT_ONLY_RAW_DATA_ALLOWED,
+    log_and_raise,
+    valid_date,
+    parse_and_split_input,
+)
+
+from openbb_terminal.menu import session
+from openbb_terminal.parent_classes import BaseController
+from openbb_terminal.rich_config import console, MenuText
+
 from openbb_terminal.forecast import (
     anom_view,
     autoarima_view,
@@ -60,19 +104,8 @@
     tft_view,
     theta_view,
     trans_view,
+    whisper_model,
 )
-from openbb_terminal.helper_funcs import (
-    EXPORT_ONLY_FIGURES_ALLOWED,
-    EXPORT_ONLY_RAW_DATA_ALLOWED,
-    NO_EXPORT,
-    check_positive,
-    check_positive_float,
-    log_and_raise,
-    valid_date,
-)
-from openbb_terminal.menu import session
-from openbb_terminal.parent_classes import BaseController
-from openbb_terminal.rich_config import MenuText, console
 
 logger = logging.getLogger(__name__)
 empty_df = pd.DataFrame()
@@ -133,6 +166,7 @@ class ForecastController(BaseController):
         "which",
         "nhits",
         "anom",
+        "whisper",
     ]
     pandas_plot_choices = [
         "line",
@@ -228,6 +262,22 @@ def get_dataset_columns(self):
             for column in dataframe.columns
         }
 
+    def parse_input(self, an_input: str) -> List:
+        """Parse controller input
+
+        Overrides the parent class function to handle YouTube video URL conventions.
+        See `BaseController.parse_input()` for details.
+        """
+        # Filtering out YouTube video parameters like "v=" and removing the domain name
+        youtube_filter = r"(youtube\.com/watch\?v=)"
+
+        custom_filters = [youtube_filter]
+
+        commands = parse_and_split_input(
+            an_input=an_input.replace("https://", ""), custom_filters=custom_filters
+        )
+        return commands
+
     def update_runtime_choices(self):
         # Load in any newly exported files
         self.DATA_FILES = forecast_model.get_default_files()
@@ -309,6 +359,9 @@ def print_help(self):
         mt.add_raw("\n")
         mt.add_info("_anomaly_")
         mt.add_cmd("anom", self.files)
+        mt.add_raw("\n")
+        mt.add_info("_misc_")
+        mt.add_cmd("whisper", WHISPER_AVAILABLE)
 
         console.print(text=mt.menu_text, menu="Forecast")
 
@@ -3219,3 +3272,99 @@ def call_anom(self, other_args: List[str]):
                 start_date=ns_parser.s_start_date,
                 end_date=ns_parser.s_end_date,
             )
+
+    @log_start_end(log=logger)
+    def call_whisper(self, other_args: List[str]):
+        """Utilize Whisper Model to transcribe a video. Currently only supports Youtube URLS"""
+        parser = argparse.ArgumentParser(
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            add_help=False,
+            prog="whisper",
+            description="""
+                Utilize Whisper Model to transcribe a video. Currently only supports Youtube URLS:
+                https://github.com/openai/whisper
+            """,
+        )
+        parser.add_argument(
+            "--video",
+            dest="video",
+            type=str,
+            default="",
+            help="video URLs to transcribe",
+        )
+        parser.add_argument(
+            "--model_name",
+            dest="model_name",
+            choices=whisper.available_models(),
+            default="base",
+            help="name of the Whisper model to use",
+        )
+        parser.add_argument(
+            "--subtitles_format",
+            dest="subtitles_format",
+            type=str,
+            choices=["vtt", "srt"],
+            help="the subtitle format to output",
+        )
+        parser.add_argument(
+            "--verbose",
+            dest="verbose",
+            type=str2bool,
+            default=False,
+            help="Whether to print out the progress and debug messages",
+        )
+        parser.add_argument(
+            "--task",
+            dest="task",
+            type=str,
+            choices=["transcribe", "translate"],
+            help="whether to perform X->X speech recognition ('transcribe') or X->English translation ('translate')",
+        )
+        parser.add_argument(
+            "--language",
+            dest="language",
+            type=str,
+            default=None,
+            choices=sorted(LANGUAGES.keys())
+            + sorted([k.title() for k in TO_LANGUAGE_CODE.keys()]),
+            help="language spoken in the audio, skip to perform language detection",
+        )
+        parser.add_argument(
+            "--breaklines",
+            dest="breaklines",
+            type=int,
+            default=0,
+            help="Whether to break lines into a bottom-heavy pyramid shape if line length exceeds N characters. 0 disables line breaking.",
+        )
+        parser.add_argument(
+            "--save",
+            dest="save",
+            type=str,
+            default=USER_FORECAST_WHISPER_DIRECTORY,
+            help="Directory to save the subtitles file",
+        )
+
+        parser = self.add_standard_args(
+            parser,
+        )
+        if other_args and "--video" not in other_args:
+            other_args.insert(0, "--video")
+        ns_parser = self.parse_known_args_and_warn(
+            parser,
+            other_args,
+        )
+
+        if ns_parser:
+            if ns_parser.save is None:
+                ns_parser.save = USER_FORECAST_WHISPER_DIRECTORY
+
+            whisper_model.transcribe_and_summarize(
+                video=ns_parser.video,
+                model_name=ns_parser.model_name,
+                subtitles_format=ns_parser.subtitles_format,
+                verbose=ns_parser.verbose,
+                task=ns_parser.task,
+                language=ns_parser.language,
+                breaklines=ns_parser.breaklines,
+                output_dir=ns_parser.save,
+            )