Merge pull request #577 from mlrun/1.6.x-dev

1.6.x dev
mlrun · Feb 29, 2024 · 8d98b67 · 8d98b67
2 parents a740a00 + ae86ca0
commit 8d98b67
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 2 deletions.
diff --git a/network-operations/01-ingest.ipynb b/network-operations/01-ingest.ipynb
@@ -160,11 +160,22 @@
    "outputs": [],
    "source": [
     "from src.generator import get_sample, config_to_static_mappings\n",
+    "import datetime \n",
+    "import pandas as pd\n",
     "\n",
     "with open(metric_configurations_path, \"r\") as fp:\n",
     "    metrics_configuration = yaml.safe_load(fp)\n",
     "\n",
-    "metrics_df, labels_df, static_df = get_sample(metrics_configuration, project=project)"
+    "# Reading perliminary dataset from database to avoid unbalanced initial data when training\n",
+    "new_timestamp = (datetime.datetime.now() - datetime.timedelta(days=1)).replace(microsecond=0)\n",
+    "\n",
+    "static_df = pd.read_parquet(mlrun.get_sample_path('data/demos-data/network-operations/static_df.parquet'))\n",
+    "\n",
+    "metrics_df = pd.read_parquet(mlrun.get_sample_path('data/demos-data/network-operations/metrics_df.parquet'))\n",
+    "metrics_df.index = metrics_df.index.set_levels([new_timestamp], level=0)\n",
+    "\n",
+    "labels_df = pd.read_parquet(mlrun.get_sample_path('data/demos-data/network-operations/labels_df.parquet'))\n",
+    "labels_df.index = labels_df.index.set_levels([new_timestamp], level=0)"
    ]
   },
   {

diff --git a/network-operations/src/generator.py b/network-operations/src/generator.py
@@ -155,7 +155,7 @@ def init_context(context):
     )
 
     static_set = fstore.get_feature_set("static")
-    static_set.ingest(static_deployment)
+    static_set.ingest(static_deployment, overwrite=False)
 
     setattr(context, "label_col_indicator", "error")
     setattr(context, "deployment_levels", ["device"])