Integration test coverage (#4969)

* Integration test coverage POC

* Update

* Integration test cleanup

* Fix

* update tests > index 36

* rename stocks tests

* Update Integration Tests

* Update Integration Test Coverage

* Update integration tests

* Linting

* Calculate parameter coverage

* Improvements

* Integration test update

* Improvements

* Improvements

* Improvements

* Improvements

* Add to controller

* Update README

* Add integration test summary

* Update integration tests

* Update integration tests

* Update algorithm

* avoid uncaught exception

* change summary

* change workflow

* another summawry change

* fix small bug when module not found

* summary table

* Refactor

* Workflow

* Breaking get_coverage_all_controllers

* Submenu coverage refactor

* Remove comments

* fix build workflow

* forgot intel

* update integration tests wf

* update windows wd (need to test)

* same

* fix integration test

* Windows patch

* Update integration-test.yml

* fix windows path

* fix windows build wf

* Update windows10_build.yml

* Linting

* fix paths

* Finishing path touches

---------

Co-authored-by: Diogo Sousa <montezdesousa@gmail.com>
Co-authored-by: luqazino@gmail.com <luqazino@gmail.com>
Co-authored-by: montezdesousa <79287829+montezdesousa@users.noreply.github.com>

.github/workflows/integration-test.yml

@@ -26,7 +26,6 @@ jobs:
       if: always()
 
     - name: Checkout code
-      if: github.event_name == 'push'
       uses: actions/checkout@v3
 
     - name: Checkout specific ref
@@ -69,15 +68,23 @@ jobs:
       run: |
         source $VENV
         python terminal.py -t | tee result.txt
-        grep "================================ integration test summary ================================" result.txt -A100 | tail -c 2000 > failed_tests.txt
+        grep "================================ Integration Test Summary ================================" result.txt -A100 | tail --bytes=2000 > summary.txt
+
+    - name: Run Integration Tests Coverage Report
+      id: integration-tests-coverage
+      run: |
+        source $VENV
+        python terminal.py -t --coverage | tee result.txt
+        echo  >> summary.txt
+        sed -n '/Integration Coverage Summary/,$p' result.txt >> summary.txt
 
     - name: Upload summary to Slack
       uses: adrey/slack-file-upload-action@master
       with:
         token: ${{ secrets.SLACK_API_TOKEN }}
         initial_comment: "Integration test summary"
         title: "Integration test summary"
-        path: failed_tests.txt
+        path: summary.txt
         channel: ${{ secrets.SLACK_CHANNEL_ID }}
 
     - name: Slack Notification - Success/Failure

.github/workflows/macos-build.yml

@@ -70,7 +70,8 @@ jobs:
           MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
           MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
           MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
-        run: | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
+        run:
+          | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
           echo "Ensuring Keychain with same name does not exist"
           rm -rf /Users/openbb/Library/Keychains/build.keychain-db
           echo "Decoding certificate"
@@ -141,16 +142,22 @@ jobs:
 
       - name: Run Integration Tests
         run: |
-            extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt
-            grep "================================ integration test summary ================================" result.txt -A100 | tail --bytes=2000 > failed_tests.txt
+          extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt
+          grep "================================ Integration Test Summary ================================" result.txt -A100 | tail --bytes=2000 > summary.txt
+
+      - name: Run Integration Tests Coverage Report
+        run: |
+          extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t --coverage | tee result.txt
+          echo  >> summary.txt
+          sed -n '/Integration Coverage Summary/,$p' result.txt >> summary.txt
 
       - name: Upload summary to Slack
         uses: adrey/slack-file-upload-action@master
         with:
           token: ${{ secrets.SLACK_API_TOKEN }}
           initial_comment: "MacOs Build Integration test summary"
           title: "MacOs Build Integration test summary"
-          path: failed_tests.txt
+          path: summary.txt
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
 
       - name: Remove OpenBB Folder
@@ -159,7 +166,7 @@ jobs:
           rm -rf ~/Desktop/OPENBB-exports
           rm -rf extract/
 
-# Job to build the MacOS Intel version of the Terminal===================================
+  # Job to build the MacOS Intel version of the Terminal===================================
   Intel-MacOs-Build:
     name: Intel MacOS Build
     runs-on: [self-hosted, macos, x64]
@@ -216,7 +223,8 @@ jobs:
           MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
           MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
           MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
-        run: | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
+        run:
+          | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
           echo "Ensuring Keychain with same name does not exist"
           rm -rf /Users/openbb/Library/Keychains/build.keychain-db
           echo "Decoding certificate"
@@ -287,16 +295,22 @@ jobs:
 
       - name: Run Integration Tests
         run: |
-            extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt
-            grep "================================ integration test summary ================================" result.txt -A100 | tail --bytes=2000 > failed_tests.txt
+          extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt
+          grep "================================ Integration Test Summary ================================" result.txt -A100 | tail --bytes=2000 > summary.txt
+
+      - name: Run Integration Tests Coverage Report
+        run: |
+          extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t --coverage | tee result.txt
+          echo  >> summary.txt
+          sed -n '/Integration Coverage Summary/,$p' result.txt >> summary.txt
 
       - name: Upload summary to Slack
         uses: adrey/slack-file-upload-action@master
         with:
           token: ${{ secrets.SLACK_API_TOKEN }}
           initial_comment: "MacOs Build Integration test summary"
           title: "Intel MacOs Build Integration test summary"
-          path: failed_tests.txt
+          path: summary.txt
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
 
       - name: Remove OpenBB Folder

.github/workflows/windows10_build.yml

@@ -105,15 +105,21 @@ jobs:
       - name: Run Integration Tests
         run: |
           dist\OpenBBTerminal\OpenBBTerminal.exe -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt
-          Select-String "================================ integration test summary ================================" result.txt -Context 0,100 | Select-Object -Last 2000 | Out-File failed_tests.txt
+          Select-String "================================ Integration Test Summary ================================" result.txt -Context 0,100 | Select-Object -Last 2000 | Out-File summary.txt
+
+      - name: Run Integration Tests Coverage Report
+        run: |
+          dist\OpenBBTerminal\OpenBBTerminal.exe -t --coverage | tee result.txt
+          echo/ >> summary.txt
+          Select-String "============================== Integration Coverage Summary ==============================" result.txt -Context 0,300 | Select-Object -Last 2000 >> summary.txt
 
       - name: Upload summary to Slack
         uses: adrey/slack-file-upload-action@master
         with:
           token: ${{ secrets.SLACK_API_TOKEN }}
           initial_comment: "Windows Build Integration test summary"
           title: "Windows Build Integration test summary"
-          path: failed_tests.txt
+          path: summary.txt
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
 
       # Cleaning ------------------------

openbb_terminal/core/integration_tests/README.md

@@ -215,6 +215,12 @@ To run the tests on installers you can use the same syntax as above, just substi
     /Full/Path/To/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t alternative/test_alt_covid.openbb
     ```
 
+- Get integration test coverage:
+
+    ```zsh
+    /Full/Path/To/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t --coverage
+    ```
+
 ## 4. Test report
 
 The console report has 3 sections: progress, failures and summary.

openbb_terminal/core/integration_tests/controllers.json

@@ -0,0 +1,26 @@
+{
+    "openbb_terminal.mutual_funds.mutual_fund_controller": "FundController",
+    "openbb_terminal.alternative.alt_controller": "AlternativeDataController",
+    "openbb_terminal.alternative.realestate.realestate_controller": "RealEstateController",
+    "openbb_terminal.alternative.oss.oss_controller": "OSSController",
+    "openbb_terminal.forex.technical_analysis.ta_controller": "TechnicalAnalysisController",
+    "openbb_terminal.stocks.research.res_controller": "ResearchController",
+    "openbb_terminal.stocks.tradinghours.tradinghours_controller": "TradingHoursController",
+    "openbb_terminal.stocks.backtesting.bt_controller": "BacktestingController",
+    "openbb_terminal.stocks.behavioural_analysis.ba_controller": "BehaviouralAnalysisController",
+    "openbb_terminal.stocks.discovery.disc_controller": "DiscoveryController",
+    "openbb_terminal.etf.discovery.disc_controller": "DiscoveryController",
+    "openbb_terminal.stocks.fundamental_analysis.fa_controller": "FundamentalAnalysisController",
+    "openbb_terminal.stocks.technical_analysis.ta_controller": "TechnicalAnalysisController",
+    "openbb_terminal.stocks.dark_pool_shorts.dps_controller": "DarkPoolShortsController",
+    "openbb_terminal.stocks.comparison_analysis.ca_controller": "ComparisonAnalysisController",
+    "openbb_terminal.fixedincome.fixedincome_controller": "FixedIncomeController",
+    "openbb_terminal.portfolio.portfolio_optimization.po_controller": "PortfolioOptimizationController",
+    "openbb_terminal.cryptocurrency.nft.nft_controller": "NFTController",
+    "openbb_terminal.cryptocurrency.technical_analysis.ta_controller": "TechnicalAnalysisController",
+    "openbb_terminal.cryptocurrency.due_diligence.dd_controller": "DueDiligenceController",
+    "openbb_terminal.etf.etf_controller": "ETFController",
+    "openbb_terminal.etf.technical_analysis.ta_controller": "TechnicalAnalysisController",
+    "openbb_terminal.reports.reports_controller": "ReportController",
+    "openbb_terminal.portfolio.portfolio_optimization.parameters.params_controller": "ParametersController"
+}

openbb_terminal/core/integration_tests/integration_controller.py

@@ -15,10 +15,19 @@
 from traceback import FrameSummary, extract_tb, format_list
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
+from matplotlib import pyplot as plt
+
 from openbb_terminal.core.config.paths import (
     MISCELLANEOUS_DIRECTORY,
     REPOSITORY_DIRECTORY,
 )
+from openbb_terminal.core.integration_tests.integration_test_coverage import (
+    get_coverage_all_controllers,
+)
+from openbb_terminal.core.integration_tests.utils import (
+    SECTION_LENGTH,
+    to_section_title,
+)
 from openbb_terminal.core.models import (
     CredentialsModel,
     PreferencesModel,
@@ -55,49 +64,9 @@
     "currency_vs",
 ]
 
-SECTION_LENGTH = 90
-STYLES = [
-    "[bold]",
-    "[/bold]",
-    "[red]",
-    "[/red]",
-    "[green]",
-    "[/green]",
-    "[bold red]",
-    "[/bold red]",
-]
 SCRIPTS_DIRECTORY = MISCELLANEOUS_DIRECTORY / "integration_tests_scripts"
 
 
-def to_section_title(title: str, char: str = "=") -> str:
-    """Format title for test mode.
-
-    Parameters
-    ----------
-    title: str
-        The title to format
-
-    Returns
-    -------
-    str
-        The formatted title
-    """
-    title = " " + title + " "
-
-    len_styles = 0
-    for style in STYLES:
-        if style in title:
-            len_styles += len(style)
-
-    n = int((SECTION_LENGTH - len(title) + len_styles) / 2)
-    formatted_title = char * n + title + char * n
-    formatted_title = formatted_title + char * (
-        SECTION_LENGTH - len(formatted_title) + len_styles
-    )
-
-    return formatted_title
-
-
 TEST_FILES = sorted(list(SCRIPTS_DIRECTORY.glob("**/*.openbb")))
 
 
@@ -537,7 +506,7 @@ def display_summary(
     """
 
     if fails:
-        console.print("\n" + to_section_title("integration test summary"))
+        console.print("\n" + to_section_title("Integration Test Summary"))
 
         for file, exception in fails.items():
             # Assuming the broken command is the last one called in the traceback
@@ -699,6 +668,14 @@ def parse_args_and_run():
         action="store_true",
         default=False,
     )
+    parser.add_argument(
+        "-c",
+        "--coverage",
+        help="Display integration test coverage.",
+        dest="coverage",
+        action="store_true",
+        default=False,
+    )
     for arg in special_arguments_values:
         parser.add_argument(
             f"--{arg}",
@@ -710,6 +687,9 @@ def parse_args_and_run():
 
     ns_parser, unknown_args = parser.parse_known_args()
 
+    if ns_parser.coverage:
+        return get_coverage_all_controllers()
+
     # Allow the tester to send a path without the -p flag
     if not ns_parser.path and unknown_args:
         ns_parser.path = [u for u in unknown_args if u[0] != "-"]
@@ -729,7 +709,7 @@ def parse_args_and_run():
     if ns_parser.list_:
         return display_available_scripts(ns_parser.path, ns_parser.skip)
 
-    run_test_session(
+    return run_test_session(
         path_list=ns_parser.path,
         skip_list=ns_parser.skip,
         special_arguments=special_args_dict,
@@ -747,7 +727,7 @@ def main():
     if "--test" in sys.argv:
         sys.argv.remove("--test")
 
-    # user
+    # User
     current_user = get_current_user()
     current_user.preferences.ENABLE_EXIT_AUTO_HELP = False
     current_user.preferences.USE_PROMPT_TOOLKIT = False
@@ -756,12 +736,15 @@ def main():
     current_user.preferences.USE_INTERACTIVE_DF = False
     set_current_user(current_user)
 
-    # system
+    # System
     set_system_variable("HEADLESS", True)
     set_system_variable("DEBUG_MODE", True)
     set_system_variable("LOG_COLLECT", False)
 
-    # run integration tests
+    # Portfolio optimization - automatically close matplotlib figures
+    plt.ion()
+
+    # Run integration tests
     parse_args_and_run()