Feature/v4 pypi

.github/workflows/platform-api-integration-test.yml

@@ -102,8 +102,8 @@ jobs:
           pip list
           uvicorn openbb_core.api.rest_api:app --host 0.0.0.0 --port 8000 --reload &
 
-      - name: Wait for 10 seconds
-        run: sleep 30
+      - name: Wait for 42 seconds
+        run: sleep 42
 
       - name: Run API Integration Tests
         run: |

.github/workflows/test_pypi_platform.yml

@@ -0,0 +1,68 @@
+name: Deploy the OpenBB Platform and the OpenBBTerminal to Test PyPI
+
+on:
+  push:
+    branches:
+      - feature/v4-pypi
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-test-pypi:
+    name: Build and publish 📦 to TestPyPI
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: OpenBBTerminal - Update version in pyproject.toml and Edit docs
+        run: |
+          sed -i 's/name = ".*"/name = "openbb-terminal-nightly"/' pyproject.toml
+          sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" pyproject.toml
+          sed -i 's|pip install openbb-terminal|pip install openbb-terminal-nightly|g' ./website/pypi.md
+
+      - name: OpenBB Platform - Update version in pyproject.toml and Edit docs
+        run: |
+          sed -i 's/name = ".*"/name = "openbb-nightly"/' openbb_platform/pyproject.toml
+          sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" openbb_platform/pyproject.toml
+
+      - name: Install pypa/build
+        run: >-
+          python -m
+          pip install
+          build
+          --user
+
+      - name: OpenBBTerminal - Build a binary wheel and a source tarball
+        run: >-
+          python -m
+          build
+          --sdist
+          --wheel
+          --outdir dist/
+          .
+
+      - name: OpenBBTerminal - Publish distribution 📦 to Test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          password: ${{ secrets.TEST_PYPI_API_TOKEN_OBB_TERMINAL_NIGHTLY }}
+          repository-url: https://test.pypi.org/legacy/
+
+      - name: OpenBB Platform - Create the dinamically generated wheel
+        run: |
+          python -m pip install poetry toml
+          python openbb_platform/dev_utils/nightly.py
+
+      - name: OpenBB Platform - Publish distribution 📦 to Test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: openbb_platform/dist/
+          password: ${{ secrets.TEST_PYPI_API_TOKEN_OBB_PLATFORM_NIGHTLY }}
+          repository-url: https://test.pypi.org/legacy/

noxfile.py

@@ -11,7 +11,11 @@
 def tests(session):
     session.install("poetry", "toml")
     session.run(
-        "python", "./openbb_platform/dev_install.py", "-e", "all", external=True
+        "python",
+        "./openbb_platform/dev_utils/dev_install.py",
+        "-e",
+        "all",
+        external=True,
     )
     session.install("pytest")
     session.install("pytest-cov")

openbb_platform/PUBLISH.md → openbb_platform/dev_utils/PUBLISH.md

@@ -6,13 +6,15 @@ Publishing checklist:
 
 1. Ensure all unit tests pass: `pytest openbb_platform -m "not integration"`
 2. Ensure all integration tests pass: `pytest openbb_platform -m integration`
-3. Change the Platform version on `openbb_platform/platform/core/openbb_core/app/constants.py`
-4. Run the publishing script: `python openbb_platform/dev_publish.py`
+3. Change the Platform version on:
+   -  `openbb_platform/platform/core/openbb_core/app/constants.py`
+   - `openbb_platform/README.md`
+4. Run the publishing script: `python openbb_platform/dev_utils/dev_publish.py`
 5. Update poetry files: `python openbb_platform/poetry_update.py`
 6. Open a PR so that changes are reflected on the main branch
 
 Finally, check if everything worked:
 
-1. Install and test the package from Pypi on a clean environment
+1. Install and test the package from Pypi on a clean environment.
 2. Check if all the `pyproject.toml` files are correct, including the `openbb_platform` one.
 3. Double check if there is any new extension or provider that needs to be added to [integration tests GitHub Action workflow](/.github/workflows/platform-api-integration-test.yml).

openbb_platform/dev_install.py → openbb_platform/dev_utils/dev_install.py

@@ -6,7 +6,7 @@
 
 import toml
 
-PLATFORM_PATH = Path(__file__).parent.resolve()
+PLATFORM_PATH = Path(__file__).parent.parent.resolve()
 LOCK = PLATFORM_PATH / "poetry.lock"
 PYPROJECT = PLATFORM_PATH / "pyproject.toml"
 
@@ -72,7 +72,9 @@ def install_local(_extras: bool = False):
 
     try:
         # we create a temporary pyproject.toml
-        PYPROJECT.write_text(TEMP_PYPROJECT)
+        with open(PYPROJECT, "w", encoding="utf-8", newline="\n") as f:
+            f.write(TEMP_PYPROJECT)
+
         CMD = [sys.executable, "-m", "poetry"]
 
         subprocess.run(  # noqa: PLW1510

openbb_platform/dev_utils/dev_publish.py

@@ -0,0 +1,39 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from dev_utils.dev_install import PLATFORM_PATH
+
+SUB_PACKAGES = ["platform/provider", "platform/core", "extensions", "providers"]
+
+CMD = [sys.executable, "-m", "poetry"]
+VERSION_BUMP_CMD = ["version", "prerelease"]
+PUBLISH_CMD = ["publish", "--build"]
+
+
+def run_cmds(directory: Path):
+    """Run the commands for publishing"""
+    print(f"Publishing: {directory.name}")  # noqa: T201
+
+    subprocess.run(CMD + VERSION_BUMP_CMD, cwd=directory, check=True)  # noqa: S603
+    subprocess.run(CMD + PUBLISH_CMD, cwd=directory, check=True)  # noqa: S603
+
+
+def publish():
+    """Publish the Platform to PyPi"""
+    for sub_path in SUB_PACKAGES:
+        for path in PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml"):
+            run_cmds(path.parent)
+
+    # openbb
+    run_cmds(PLATFORM_PATH)
+
+
+if __name__ == "__main__":
+    raise Exception(
+        "If you're ar running this script for the first time,"
+        "ensure you have changed `VERSION` on System Settings "
+        "before you publish the `openbb-core` package to Pypi."
+    )
+
+    publish()

openbb_platform/dev_utils/nightly.py

@@ -0,0 +1,83 @@
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+import toml
+
+PLATFORM_PATH = Path(__file__).parent.parent.resolve()
+PYPROJECT = PLATFORM_PATH / "pyproject.toml"
+
+SUB_PACKAGES = {}
+DEPENDENCIES: Dict[str, Any] = {}
+PLUGINS = {"openbb_core_extension": {}, "openbb_provider_extension": {}}
+CMD = [sys.executable, "-m", "poetry", "build"]
+
+
+PYPROJECT_TOML = toml.load(PYPROJECT)
+POETRY_DICT: Dict[str, dict] = PYPROJECT_TOML["tool"]["poetry"]
+POETRY_DICT.pop("extras", None)
+
+
+original_pyproject = PYPROJECT.read_text()
+
+
+def gather_metadata(sub_path: str):
+    """Gather metadata from the pyproject.toml files.
+
+    Parameters
+    ----------
+    sub_path : str
+        The path to the sub packages.
+    """
+    for path in PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml"):
+        pyproject_toml = toml.load(path)
+        poetry_dict: Dict[str, dict] = pyproject_toml["tool"]["poetry"]
+        package_name = poetry_dict["packages"][0]["include"]
+
+        for extension in list(PLUGINS.keys()):
+            PLUGINS[extension].update(poetry_dict.get("plugins", {}).get(extension, {}))
+
+        DEPENDENCIES.update(pyproject_toml["tool"]["poetry"]["dependencies"])
+        SUB_PACKAGES[package_name] = path.relative_to(PLATFORM_PATH).parent.as_posix()
+
+
+def build():
+    """Build the Platform package."""
+    for sub_path in ["platform", "providers", "extensions"]:
+        gather_metadata(sub_path)
+
+    # need to pop these from the dependencies
+    DEPENDENCIES.pop("openbb-core", None)
+    DEPENDENCIES.pop("openbb-provider", None)
+
+    # add the sub packages
+    for package_name, path in SUB_PACKAGES.items():
+        POETRY_DICT["packages"].append({"include": package_name, "from": f"./{path}"})
+
+    # add the plugins extensions
+    for extension, plugins in PLUGINS.items():
+        POETRY_DICT.setdefault("plugins", {}).setdefault(extension, {}).update(plugins)
+
+    # update the dependencies and platform poetry dict
+    POETRY_DICT["dependencies"] = DEPENDENCIES
+    PYPROJECT_TOML["tool"]["poetry"] = POETRY_DICT
+
+    temp_pyproject = toml.dumps(PYPROJECT_TOML)
+
+    try:
+        with open(PYPROJECT, "w", encoding="utf-8", newline="\n") as f:
+            f.write(temp_pyproject)
+
+        subprocess.run(CMD, cwd=PLATFORM_PATH, check=True)  # noqa: S603,PLW1510
+    except (Exception, KeyboardInterrupt) as e:
+        print(e)  # noqa: T201
+        print("Restoring pyproject.toml")  # noqa: T201
+
+    # we restore the original pyproject.toml
+    with open(PYPROJECT, "w", encoding="utf-8", newline="\n") as f:
+        f.write(original_pyproject)
+
+
+if __name__ == "__main__":
+    build()

openbb_platform/poetry_update.py → openbb_platform/dev_utils/poetry_update.py

@@ -1,27 +1,27 @@
-import os
+import subprocess
 import sys
-from pathlib import Path
 from typing import List, Optional
 
+from dev_utils.dev_install import PLATFORM_PATH
+
 
 def update_lock_file(packages: Optional[List[str]] = None) -> None:
     """Update the poetry lock files in all the packages or in the given packages."""
-    for path in Path.cwd().rglob("pyproject.toml"):
+    for path in PLATFORM_PATH.rglob("pyproject.toml"):
         if packages and path.parent.name not in packages:
             print(f"Skipping {path.parent.name}")  # noqa: T201
             continue
 
         print(f"\nUpdating lock file in {path.parent.name}")  # noqa: T201
-        os.chdir(path.parent)
-        os.system("poetry lock --no-update")  # noqa: S605, S607
+        CMD = [sys.executable, "-m", "poetry", "lock", "--no-update"]
+        subprocess.run(CMD, cwd=path.parent, check=True)  # noqa: S603,PLW1510
 
-        # Fix line endings and encoding
         lock_file = path.parent / "poetry.lock"
-        lock_file.write_text(
-            lock_file.read_text(),
-            encoding="utf-8",
-            newline="\n",
-        )
+        lock_text = lock_file.read_text()
+
+        # Fix line endings and encoding
+        with open(lock_file, "w", encoding="utf-8", newline="\n") as f:
+            f.write(lock_text)
 
 
 if __name__ == "__main__":

website/content/platform/faqs/installation_updates.md

@@ -70,7 +70,7 @@ poetry install -E all
 The nightly build can be installed with:
 
 ```console
-pip install openbb-nightly[all]
+pip install openbb-terminal-nightly[all]
 ```
 
 **Note**: This version may not be stable and should not be used in a production setting.

website/versioned_docs/version-v3/sdk/faqs/installation_updates.md

@@ -70,7 +70,7 @@ poetry install -E all
 The nightly build can be installed with:
 
 ```console
-pip install openbb-nightly[all]
+pip install openbb-terminal-nightly[all]
 ```
 
 **Note**: This version may not be stable and should not be used in a production setting.