source code: Handle block comments

fennel/client_tests/test_dataset.py

@@ -1822,18 +1822,24 @@ class PositiveRatingActivity:
     )
     t: datetime
 
+    # fmt: off
     @pipeline
     @inputs(RatingActivity)
     def filter_positive_ratings(cls, rating: Dataset):
         filtered_ds = rating.filter(lambda df: df["rating"] >= 3.5)
+#       This is a random comment
         filter2 = filtered_ds.filter(
             lambda df: df["movie"].isin(["Jumanji", "Titanic", "RaOne"])
         )
+        # a = b + 2
         return filter2.groupby("movie").aggregate(
             Count(window=Continuous("forever"), into_field=str(cls.cnt_rating)),
         )
 
 
+# fmt: on
+
+
 class TestBasicFilter(unittest.TestCase):
     @pytest.mark.integration
     @mock

fennel/internal_lib/to_proto/source_code.py

@@ -46,8 +46,9 @@ def get_featureset_core_code(
         if not extractor.func:
             continue
         extractor_code = fennel_get_source(extractor.func)
-        extractor_code = indent(dedent(extractor_code), " " * 4)
+        extractor_code_indented = indent(dedent(extractor_code), " " * 4)
         # Delete extractor code from source_code
+        source_code = source_code.replace(extractor_code_indented, "")
         source_code = source_code.replace(extractor_code, "")
 
     # If python version 3.8 or below add @feature decorator
@@ -187,12 +188,12 @@ def get_dataset_core_code(dataset: Dataset) -> str:
     source_code = fennel_get_source(dataset.__fennel_original_cls__)
     for pipeline in dataset._pipelines:
         pipeline_code = fennel_get_source(pipeline.func)
-        pipeline_code = indent(dedent(pipeline_code), " " * 4)
+        pipeline_code_indented = indent(dedent(pipeline_code), " " * 4)
         # Delete pipeline code from source_code
+        source_code = source_code.replace(pipeline_code_indented, "")
         source_code = source_code.replace(pipeline_code, "")
     # Delete decorator @source() and @sink from source_code using regex
     source_code = remove_decorators(source_code, ["source", "sink"])
-
     # If python version 3.8 or below add @dataset decorator
     if sys.version_info < (3, 9):
         source_code = f"@dataset\n{dedent(source_code)}"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fennel-ai"
-version = "1.4.6"
+version = "1.4.7"
 description = "The modern realtime feature engineering platform"
 authors = ["Fennel AI <developers@fennel.ai>"]
 packages = [{ include = "fennel" }]