Kedro Viz Static Website Hosting on GCP (#1711)

* CLI command kedro viz build added * Lint fix * lint fix * Lint fix * add mypy ignore * Missing build file added * Lint error fix * BaseDeployer class added * Unused code removed * Fix lint issue * azure deploy initial draft Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * added base_deployer * add deployer factory * partial working draft Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * Test and comments of deployers updated * test draft Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * fix lint Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * remove circular dependency Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * fix lint Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * revert back consent Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * initial draft Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * minor updates Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * update pytests Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * add pytest for azure shareableviz Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * refactor and add timeout Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * refactor cli Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * update pytest Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * add release note Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * fix flaky test Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * fix PR comments and flaky test Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * testing flaky c y test Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * remove flaky test Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * add pytest for gcp Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * fix gcp pytest coverage Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * fix lint Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * update pytest Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> * revert file permission change --------- Signed-off-by: ravi-kumar-pilla <ravi_kumar_pilla@mckinsey.com> Co-authored-by: Jitendra Gundaniya <jitendra_gundaniya@mckinsey.com> Co-authored-by: rashidakanchwala <rashida_kanchwala@mckinsey.com>
kedro-org · Feb 13, 2024 · 8f5a152 · 8f5a152
1 parent 25a7aff
commit 8f5a152
Show file tree

Hide file tree

Showing 10 changed files with 180 additions and 10 deletions.
diff --git a/RELEASE.md b/RELEASE.md
@@ -9,7 +9,7 @@ Please follow the established format:
 
 ## Major features and improvements
 
-- Kedro Viz Static Website hosting on Azure. (#1708)
+- Extend support for 'Publishing and sharing Kedro-viz' to Azure and GCP. (#1708, #1711)
 
 # Upcoming release
 

diff --git a/package/kedro_viz/constants.py b/package/kedro_viz/constants.py
@@ -12,4 +12,4 @@
 
 VIZ_DEPLOY_TIME_LIMIT = 300
 
-SHAREABLEVIZ_SUPPORTED_PLATFORMS = ["aws", "azure"]
+SHAREABLEVIZ_SUPPORTED_PLATFORMS = ["aws", "azure", "gcp"]
diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py
@@ -67,7 +67,7 @@ def _write_heap_injected_index(self, html_content):
             with open(temp_file_path, "w", encoding="utf-8") as temp_index_file:
                 temp_index_file.write(html_content)
 
-            self._fs.put(temp_file_path, f"{self._path}/")
+            self._fs.put(temp_file_path, f"{self._path}/", content_type="text/html")
 
     def _upload_static_files(self, html_dir: Path):
         """Upload static HTML files to Build."""

diff --git a/package/kedro_viz/integrations/deployment/deployer_factory.py b/package/kedro_viz/integrations/deployment/deployer_factory.py
@@ -4,6 +4,7 @@
 from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS
 from kedro_viz.integrations.deployment.aws_deployer import AWSDeployer
 from kedro_viz.integrations.deployment.azure_deployer import AzureDeployer
+from kedro_viz.integrations.deployment.gcp_deployer import GCPDeployer
 from kedro_viz.integrations.deployment.local_deployer import LocalDeployer
 
 
@@ -13,11 +14,14 @@ class DeployerFactory:
     @staticmethod
     def create_deployer(platform, endpoint=None, bucket_name=None):
         """Instantiate Kedro-viz deployer classes"""
-        if platform.lower() == "aws":
+        platform_name = platform.lower()
+        if platform_name == "aws":
             return AWSDeployer(endpoint, bucket_name)
-        if platform.lower() == "azure":
+        if platform_name == "azure":
             return AzureDeployer(endpoint, bucket_name)
-        if platform.lower() == "local":
+        if platform_name == "gcp":
+            return GCPDeployer(endpoint, bucket_name)
+        if platform_name == "local":
             return LocalDeployer()
         raise ValueError(
             f"Invalid platform '{platform}' specified. \n"

diff --git a/package/kedro_viz/integrations/deployment/gcp_deployer.py b/package/kedro_viz/integrations/deployment/gcp_deployer.py
@@ -0,0 +1,69 @@
+"""`kedro_viz.integrations.deployment.gcp_deployer` defines
+deployment class for Google Cloud Storage Bucket"""
+import glob
+import logging
+import mimetypes
+from pathlib import Path
+
+import fsspec
+
+from kedro_viz import __version__
+from kedro_viz.integrations.deployment.base_deployer import BaseDeployer
+
+_GCP_PROTOCOL = "gcs"
+logger = logging.getLogger(__name__)
+
+
+class GCPDeployer(BaseDeployer):
+    """A class to handle the deployment of Kedro-viz to Google Cloud Storage Bucket.
+
+    Attributes:
+        _endpoint (str): GCP endpoint of the hosted site.
+        _bucket_name (str): Name of the GCP storage bucket.
+        _path (str): GCP protocol with bucket name.
+        _fs (fsspec.filesystem): Filesystem for GCP protocol.
+    """
+
+    def __init__(self, endpoint, bucket_name):
+        """Initialize GCPDeployer with endpoint and bucket name.
+
+        Args:
+            endpoint (str): GCP endpoint of the hosted site.
+            bucket_name (str): Name of the GCP storage bucket.
+        """
+        super().__init__()
+        self._endpoint = endpoint
+        self._bucket_name = bucket_name
+        self._path = f"{_GCP_PROTOCOL}://{bucket_name}"
+        self._fs = fsspec.filesystem(_GCP_PROTOCOL)
+
+    def _upload_static_files(self, html_dir: Path):
+        logger.debug("Uploading static html files to %s.", self._path)
+        try:
+            file_list = glob.glob(f"{str(html_dir)}/**/*", recursive=True)
+
+            for local_file_path in file_list:
+                content_type, _ = mimetypes.guess_type(local_file_path)
+
+                # ignore directories
+                if content_type is None:  # pragma: no cover
+                    continue
+
+                relative_path = local_file_path[len(str(html_dir)) + 1 :]
+                remote_file_path = f"{self._path}/{relative_path}"
+
+                # Read the contents of the local file
+                with open(local_file_path, "rb") as file:
+                    content = file.read()
+
+                self._fs.write_bytes(
+                    path=remote_file_path,
+                    value=content,
+                    content_type=content_type,
+                )
+
+            self._ingest_heap_analytics()
+
+        except Exception as exc:  # pragma: no cover
+            logger.exception("Upload failed: %s ", exc)
+            raise exc
diff --git a/package/setup.py b/package/setup.py
@@ -60,5 +60,6 @@
         ],
         "aws": ["s3fs>=2021.4"],
         "azure": ["adlfs>=2021.4"],
+        "gcp": ["gcsfs>=2021.4"],
     },
 )
diff --git a/package/test_requirements.txt b/package/test_requirements.txt
@@ -40,5 +40,6 @@ types-toml==0.1.1
 types-ujson==0.1.0
 
 #shareableviz
-s3fs>=2021.4, <2024.1
-adlfs>=2021.4, <2024.1
+s3fs>=2021.4
+adlfs>=2021.4
+gcsfs>=2021.4
diff --git a/package/tests/test_integrations/test_deployer_factory.py b/package/tests/test_integrations/test_deployer_factory.py
@@ -6,14 +6,16 @@
 from kedro_viz.integrations.deployment.aws_deployer import AWSDeployer
 from kedro_viz.integrations.deployment.azure_deployer import AzureDeployer
 from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory
+from kedro_viz.integrations.deployment.gcp_deployer import GCPDeployer
 from kedro_viz.integrations.deployment.local_deployer import LocalDeployer
 
 
 @pytest.mark.parametrize(
     "platform, endpoint, bucket_name, deployer_class",
     [
-        ("aws", "http://mocked-url.com", "s3://shareableviz", AWSDeployer),
-        ("azure", "http://mocked-url.com", "abfs://shareableviz", AzureDeployer),
+        ("aws", "http://mocked-url.com", "shareableviz", AWSDeployer),
+        ("azure", "http://mocked-url.com", "shareableviz", AzureDeployer),
+        ("gcp", "http://mocked-url.com", "shareableviz", GCPDeployer),
     ],
 )
 def test_create_deployer(platform, endpoint, bucket_name, deployer_class):

diff --git a/package/tests/test_integrations/test_gcp_deployer.py b/package/tests/test_integrations/test_gcp_deployer.py
@@ -0,0 +1,59 @@
+import pytest
+
+from kedro_viz import __version__
+from kedro_viz.integrations.deployment.gcp_deployer import GCPDeployer
+
+
+# Test the GCPDeployer class
+@pytest.fixture
+def endpoint():
+    return "http://34.120.87.227/"
+
+
+@pytest.fixture
+def bucket_name():
+    return "test-bucket"
+
+
+@pytest.fixture
+def mock_file_system(mocker):
+    yield mocker.patch("fsspec.filesystem")
+
+
+class TestGCPDeployer:
+    def test_deploy(self, endpoint, bucket_name, mocker):
+        deployer = GCPDeployer(endpoint, bucket_name)
+
+        mocker.patch.object(deployer, "_upload_api_responses")
+        mocker.patch.object(deployer, "_upload_static_files")
+        mocker.patch.object(deployer, "_upload_deploy_viz_metadata_file")
+
+        deployer.deploy()
+
+        deployer._upload_api_responses.assert_called_once()
+        deployer._upload_static_files.assert_called_once()
+        deployer._upload_deploy_viz_metadata_file.assert_called_once()
+
+    def test_upload_static_files(
+        self, endpoint, bucket_name, tmp_path, mocker, mock_file_system
+    ):
+        deployer = GCPDeployer(endpoint, bucket_name)
+        mock_ingest_heap_analytics = mocker.patch.object(
+            deployer, "_ingest_heap_analytics"
+        )
+        mock_html_content = "<html><body>Test Content</body></html>"
+
+        # Create a temporary HTML file with some test content
+        temp_file_path = tmp_path / "test_file.html"
+        with open(temp_file_path, "w", encoding="utf-8") as temp_file:
+            temp_file.write(mock_html_content)
+
+        with mocker.patch("mimetypes.guess_type", return_value=("text/html", None)):
+            deployer._upload_static_files(tmp_path)
+            deployer._fs.write_bytes.assert_called_once_with(
+                path=f"gcs://{bucket_name}/test_file.html",
+                value=mock_html_content.encode("utf-8"),
+                content_type="text/html",
+            )
+
+            mock_ingest_heap_analytics.assert_called_once()
diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py
@@ -334,6 +334,23 @@ def test_viz_command_group(mocker, mock_click_echo):
                 "bucket_name": "example-bucket",
             },
         ),
+        (
+            [
+                "viz",
+                "deploy",
+                "--platform",
+                "gcp",
+                "--endpoint",
+                "http://34.120.87.227/",
+                "--bucket-name",
+                "example-bucket",
+            ],
+            {
+                "platform": "gcp",
+                "endpoint": "http://34.120.87.227/",
+                "bucket_name": "example-bucket",
+            },
+        ),
     ],
 )
 def test_viz_deploy_valid_endpoint_and_bucket(command_options, deployer_args, mocker):
@@ -433,6 +450,12 @@ def test_successful_build_with_existing_static_files(mocker):
             "example-bucket",
             1,
         ),
+        (
+            "gcp",
+            "http://34.120.87.227/",
+            "example-bucket",
+            1,
+        ),
         ("local", None, None, 1),
         ("azure", "https://example-bucket.web.core.windows.net", "example-bucket", 0),
         (
@@ -441,6 +464,12 @@ def test_successful_build_with_existing_static_files(mocker):
             "example-bucket",
             0,
         ),
+        (
+            "gcp",
+            "http://34.120.87.227/",
+            "example-bucket",
+            0,
+        ),
         ("local", None, None, 0),
     ],
 )
@@ -509,6 +538,11 @@ def test_create_shareableviz_process(
             "http://example-bucket.s3-website.us-east-2.amazonaws.com/",
             "example-bucket",
         ),
+        (
+            "gcp",
+            "http://34.120.87.227/",
+            "example-bucket",
+        ),
         ("local", None, None),
     ],
 )
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,4 @@

		VIZ_DEPLOY_TIME_LIMIT = 300

		SHAREABLEVIZ_SUPPORTED_PLATFORMS = ["aws", "azure"]
		SHAREABLEVIZ_SUPPORTED_PLATFORMS = ["aws", "azure", "gcp"]