[Feature] - Create repo assets directory (OpenBB-finance#6384)

* feat: create scripts to generate repo assets

* move script

* publish.md

* deletes unmaintained .md files

* ruff

* pylint

* fix: website urls

* fix: website urls

* rename script

* renames

* create folder

* mypy

* PyDocstyle

* fix: descriptions & websites

* camelCase

* change json structure

* logo url

* reprName

* finra

* logos

* logo

* logo

* logo

* logo

* pylint

---------

Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>

assets/README.md

@@ -0,0 +1,5 @@
+# Assets
+
+This folder should hold assets read by OpenBB applications, such as OpenBB Hub or marketing website.
+
+The goal is to be more explicit about which assets are being used externally and cannot be deleted before checking where they are used.

assets/extensions/obbject.json

@@ -0,0 +1,6 @@
+[
+    {
+        "packageName": "openbb-charting",
+        "description": "Create custom charts from OBBject data."
+    }
+]

assets/extensions/router.json

@@ -0,0 +1,58 @@
+[
+    {
+        "packageName": "openbb-commodity",
+        "description": "Commodity market data."
+    },
+    {
+        "packageName": "openbb-crypto",
+        "description": "Cryptocurrency market data."
+    },
+    {
+        "packageName": "openbb-currency",
+        "description": "Foreign exchange (FX) market data."
+    },
+    {
+        "packageName": "openbb-derivatives",
+        "description": "Derivatives market data."
+    },
+    {
+        "packageName": "openbb-econometrics",
+        "description": "Econometrics analysis tools."
+    },
+    {
+        "packageName": "openbb-economy",
+        "description": "Economic data."
+    },
+    {
+        "packageName": "openbb-equity",
+        "description": "Equity market data."
+    },
+    {
+        "packageName": "openbb-etf",
+        "description": "Exchange Traded Funds market data."
+    },
+    {
+        "packageName": "openbb-fixedincome",
+        "description": "Fixed Income market data."
+    },
+    {
+        "packageName": "openbb-index",
+        "description": "Indices data."
+    },
+    {
+        "packageName": "openbb-news",
+        "description": "Financial market news data."
+    },
+    {
+        "packageName": "openbb-quantitative",
+        "description": "Quantitative analysis tools."
+    },
+    {
+        "packageName": "openbb-regulators",
+        "description": "Financial market regulators data."
+    },
+    {
+        "packageName": "openbb-technical",
+        "description": "Technical Analysis tools."
+    }
+]

assets/scripts/generate_extension_data.py

@@ -0,0 +1,115 @@
+"""Generate assets from modules."""
+
+from importlib import import_module
+from json import dump
+from pathlib import Path
+from typing import Any, Dict, List
+
+from poetry.core.pyproject.toml import PyProjectTOML
+
+THIS_DIR = Path(__file__).parent
+PROVIDERS_PATH = Path(THIS_DIR, "..", "..", "openbb_platform/providers")
+EXTENSIONS_PATH = Path(THIS_DIR, "..", "..", "openbb_platform/extensions")
+OBBJECT_EXTENSIONS_PATH = Path(
+    THIS_DIR, "..", "..", "openbb_platform/obbject_extensions"
+)
+
+
+def to_title(string: str) -> str:
+    """Format string to title."""
+    return " ".join(string.split("_")).title()
+
+
+def get_packages(path: Path, plugin_key: str) -> Dict[str, Any]:
+    """Get packages."""
+    SKIP = ["tests", "__pycache__"]
+    folders = [f for f in path.glob("*") if f.is_dir() and f.stem not in SKIP]
+    packages: Dict[str, Any] = {}
+    for f in folders:
+        pyproject = PyProjectTOML(Path(f, "pyproject.toml"))
+        poetry = pyproject.data["tool"]["poetry"]
+        name = poetry["name"]
+        plugin = poetry.get("plugins", {}).get(plugin_key)
+        packages[name] = list(plugin.values())[0] if plugin else ""
+    return packages
+
+
+def write(filename: str, data: Any):
+    """Write to json."""
+    with open(Path(THIS_DIR, "..", "extensions", f"{filename}.json"), "w") as json_file:
+        dump(data, json_file, indent=4)
+
+
+def to_camel(string: str):
+    """Convert string to camel case."""
+    components = string.split("_")
+    return components[0] + "".join(x.title() for x in components[1:])
+
+
+def createItem(package_name: str, obj: object, attrs: List[str]) -> Dict[str, str]:
+    """Create dictionary item from object attributes."""
+    item = {"packageName": package_name}
+    item.update(
+        {to_camel(a): getattr(obj, a) for a in attrs if getattr(obj, a) is not None}
+    )
+    return item
+
+
+def generate_provider_extensions() -> None:
+    """Generate providers_extensions.json."""
+    packages = get_packages(PROVIDERS_PATH, "openbb_provider_extension")
+    data: List[Dict[str, str]] = []
+    attrs = [
+        "repr_name",
+        "description",
+        "credentials",
+        "v3_credentials",
+        "website",
+        "instructions",
+        "logo_url",
+    ]
+
+    for pkg_name, plugin in sorted(packages.items()):
+        file_obj = plugin.split(":")
+        if len(file_obj) == 2:
+            file, obj = file_obj[0], file_obj[1]
+            module = import_module(file)
+            provider_obj = getattr(module, obj)
+            data.append(createItem(pkg_name, provider_obj, attrs))
+    write("provider", data)
+
+
+def generate_router_extensions() -> None:
+    """Generate router_extensions.json."""
+    packages = get_packages(EXTENSIONS_PATH, "openbb_core_extension")
+    data: List[Dict[str, str]] = []
+    attrs = ["description"]
+    for pkg_name, plugin in sorted(packages.items()):
+        file_obj = plugin.split(":")
+        if len(file_obj) == 2:
+            file, obj = file_obj[0], file_obj[1]
+            module = import_module(file)
+            router_obj = getattr(module, obj)
+            data.append(createItem(pkg_name, router_obj, attrs))
+    write("router", data)
+
+
+def generate_obbject_extensions() -> None:
+    """Generate obbject_extensions.json."""
+    packages = get_packages(OBBJECT_EXTENSIONS_PATH, "openbb_obbject_extension")
+    data: List[Dict[str, str]] = []
+    attrs = ["description"]
+    for pkg_name, plugin in sorted(packages.items()):
+        file_obj = plugin.split(":")
+        if len(file_obj) == 2:
+            file, obj = file_obj[0], file_obj[1]
+            module = import_module(file)
+            ext_obj = getattr(module, obj)
+            data.append(createItem(pkg_name, ext_obj, attrs))
+    write("obbject", data)
+
+
+if __name__ == "__main__":
+    generate_provider_extensions()
+    generate_router_extensions()
+    generate_obbject_extensions()

build/pypi/openbb_platform/PUBLISH.md

@@ -55,6 +55,7 @@
 
 1. Install the packages on Google Colaboratory via PyPi and test to check if everything is working as expected.
 2. Install the packages in a new environment locally via PyPi and test to check if everything is working as expected.
-3. Open a new PR with the `release/<package>-<version>` branch pointing to the `develop` branch.
-4. Merge the `release/<package>-<version>` branch to the `develop` branch.
-5. If any bugs are encountered, create a new branch - `hotfix` for `main` and `bugfix` for `develop` and merge them accordingly.
+3. Regenerate assets for external use by running `python assets/scripts/generate_extension_data.py`
+4. Open a new PR with the `release/<package>-<version>` branch pointing to the `develop` branch.
+5. Merge the `release/<package>-<version>` branch to the `develop` branch.
+6. If any bugs are encountered, create a new branch - `hotfix` for `main` and `bugfix` for `develop` and merge them accordingly.

openbb_platform/core/openbb_core/app/model/extension.py

@@ -15,6 +15,7 @@ def __init__(
         self,
         name: str,
         credentials: Optional[List[str]] = None,
+        description: Optional[str] = None,
     ) -> None:
         """Initialize the extension.
 
@@ -24,9 +25,12 @@ def __init__(
             Name of the extension.
         credentials : Optional[List[str]], optional
             List of required credentials, by default None
+        description: Optional[str]
+            Extension description.
         """
         self.name = name
         self.credentials = credentials or []
+        self.description = description
 
     @property
     def obbject_accessor(self) -> Callable:

openbb_platform/core/openbb_core/provider/abstract/provider.py

@@ -8,13 +8,18 @@
 class Provider:
     """Serves as provider extension entry point and must be created by each provider."""
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         name: str,
         description: str,
         website: Optional[str] = None,
         credentials: Optional[List[str]] = None,
         fetcher_dict: Optional[Dict[str, Type[Fetcher]]] = None,
+        repr_name: Optional[str] = None,
+        v3_credentials: Optional[List[str]] = None,
+        instructions: Optional[str] = None,
+        logo_url: Optional[str] = None,
     ) -> None:
         """Initialize the provider.
 
@@ -26,10 +31,18 @@ def __init__(
             Description of the provider.
         website : Optional[str]
             Website of the provider, by default None.
-        credentials : Optional[List[str]], optional
-            List of required credentials, by default None
+        credentials : Optional[List[str]]
+            List of required credentials, by default None.
         fetcher_dict : Optional[Dict[str, Type[Fetcher]]]
             Dictionary of fetchers, by default None.
+        repr_name: Optional[str]
+            Full name of the provider, by default None.
+        v3_credentials: Optional[List[str]]
+            List of corresponding v3 credentials, by default None.
+        instructions: Optional[str]
+            Instructions on how to setup the provider. For example, how to get an API key.
+        logo_url: Optional[str]
+            Provider logo URL.
         """
         self.name = name
         self.description = description
@@ -41,3 +54,7 @@ def __init__(
             self.credentials = []
             for c in credentials:
                 self.credentials.append(f"{self.name.lower()}_{c}")
+        self.repr_name = repr_name
+        self.v3_credentials = v3_credentials
+        self.instructions = instructions
+        self.logo_url = logo_url