Merge pull request #187 from george0st/change

Focus on None value in parquet also
george0st · Jun 22, 2024 · 32e494b · 32e494b
2 parents 295c751 + 67fb250
commit 32e494b
Show file tree

Hide file tree

Showing 62 changed files with 627 additions and 583 deletions.
diff --git a/01-model/model.json b/01-model/model.json
@@ -3,7 +3,7 @@
   "description": "The machine learning meta-model with synthetic data (useful for MLOps/feature store), part of the quality gate concept.",
   "kind": "model",
   "spec": {
-    "version": "0.3.4",
+    "version": "0.3.5",
     "CSV_SEPARATOR": ",",
     "CSV_DECIMAL": ".",
     "NONE_VALUES": true,

diff --git a/02-data/01-size-100/01-basic-party.csv.gz b/02-data/01-size-100/01-basic-party.csv.gz
diff --git a/02-data/01-size-100/01-basic-party.parquet b/02-data/01-size-100/01-basic-party.parquet
diff --git a/02-data/01-size-100/01-basic_party.csv.gz b/02-data/01-size-100/01-basic_party.csv.gz
diff --git a/02-data/01-size-100/01-basic_party.parquet b/02-data/01-size-100/01-basic_party.parquet
diff --git a/02-data/01-size-100/02-basic-contact.csv.gz b/02-data/01-size-100/02-basic-contact.csv.gz
diff --git a/02-data/01-size-100/02-basic-contact.parquet b/02-data/01-size-100/02-basic-contact.parquet
diff --git a/02-data/01-size-100/02-basic_contact.csv.gz b/02-data/01-size-100/02-basic_contact.csv.gz
diff --git a/02-data/01-size-100/02-basic_contact.parquet b/02-data/01-size-100/02-basic_contact.parquet
diff --git a/02-data/01-size-100/03-basic-relation.csv.gz b/02-data/01-size-100/03-basic-relation.csv.gz
diff --git a/02-data/01-size-100/03-basic-relation.parquet b/02-data/01-size-100/03-basic-relation.parquet
diff --git a/02-data/01-size-100/03-basic_relation.csv.gz b/02-data/01-size-100/03-basic_relation.csv.gz
diff --git a/02-data/01-size-100/03-basic_relation.parquet b/02-data/01-size-100/03-basic_relation.parquet
diff --git a/02-data/01-size-100/04-basic-account.csv.gz b/02-data/01-size-100/04-basic-account.csv.gz
diff --git a/02-data/01-size-100/04-basic-account.parquet b/02-data/01-size-100/04-basic-account.parquet
diff --git a/02-data/01-size-100/04-basic_account.csv.gz b/02-data/01-size-100/04-basic_account.csv.gz
diff --git a/02-data/01-size-100/04-basic_account.parquet b/02-data/01-size-100/04-basic_account.parquet
diff --git a/02-data/01-size-100/05-basic-transaction.csv.gz b/02-data/01-size-100/05-basic-transaction.csv.gz
diff --git a/02-data/01-size-100/05-basic-transaction.parquet b/02-data/01-size-100/05-basic-transaction.parquet
diff --git a/02-data/01-size-100/05-basic_transaction.csv.gz b/02-data/01-size-100/05-basic_transaction.csv.gz
diff --git a/02-data/01-size-100/05-basic_transaction.parquet b/02-data/01-size-100/05-basic_transaction.parquet
diff --git a/02-data/01-size-100/06-basic-event.csv.gz b/02-data/01-size-100/06-basic-event.csv.gz
diff --git a/02-data/01-size-100/06-basic-event.parquet b/02-data/01-size-100/06-basic-event.parquet
diff --git a/02-data/01-size-100/06-basic_event.csv.gz b/02-data/01-size-100/06-basic_event.csv.gz
diff --git a/02-data/01-size-100/06-basic_event.parquet b/02-data/01-size-100/06-basic_event.parquet
diff --git a/02-data/01-size-100/07-basic-communication.csv.gz b/02-data/01-size-100/07-basic-communication.csv.gz
diff --git a/02-data/01-size-100/07-basic-communication.parquet b/02-data/01-size-100/07-basic-communication.parquet
diff --git a/02-data/01-size-100/07-basic_communication.csv.gz b/02-data/01-size-100/07-basic_communication.csv.gz
diff --git a/02-data/01-size-100/07-basic_communication.parquet b/02-data/01-size-100/07-basic_communication.parquet
diff --git a/02-data/02-size-1K/01-basic-party.csv.gz b/02-data/02-size-1K/01-basic-party.csv.gz
diff --git a/02-data/02-size-1K/01-basic-party.parquet b/02-data/02-size-1K/01-basic-party.parquet
diff --git a/02-data/02-size-1K/01-basic_party.csv.gz b/02-data/02-size-1K/01-basic_party.csv.gz
diff --git a/02-data/02-size-1K/01-basic_party.parquet b/02-data/02-size-1K/01-basic_party.parquet
diff --git a/02-data/02-size-1K/02-basic-contact.csv.gz b/02-data/02-size-1K/02-basic-contact.csv.gz
diff --git a/02-data/02-size-1K/02-basic-contact.parquet b/02-data/02-size-1K/02-basic-contact.parquet
diff --git a/02-data/02-size-1K/02-basic_contact.csv.gz b/02-data/02-size-1K/02-basic_contact.csv.gz
diff --git a/02-data/02-size-1K/02-basic_contact.parquet b/02-data/02-size-1K/02-basic_contact.parquet
diff --git a/02-data/02-size-1K/03-basic-relation.csv.gz b/02-data/02-size-1K/03-basic-relation.csv.gz
diff --git a/02-data/02-size-1K/03-basic-relation.parquet b/02-data/02-size-1K/03-basic-relation.parquet
diff --git a/02-data/02-size-1K/03-basic_relation.csv.gz b/02-data/02-size-1K/03-basic_relation.csv.gz
diff --git a/02-data/02-size-1K/03-basic_relation.parquet b/02-data/02-size-1K/03-basic_relation.parquet
diff --git a/02-data/02-size-1K/04-basic-account.csv.gz b/02-data/02-size-1K/04-basic-account.csv.gz
diff --git a/02-data/02-size-1K/04-basic-account.parquet b/02-data/02-size-1K/04-basic-account.parquet
diff --git a/02-data/02-size-1K/04-basic_account.csv.gz b/02-data/02-size-1K/04-basic_account.csv.gz
diff --git a/02-data/02-size-1K/04-basic_account.parquet b/02-data/02-size-1K/04-basic_account.parquet
diff --git a/02-data/02-size-1K/05-basic-transaction.csv.gz b/02-data/02-size-1K/05-basic-transaction.csv.gz
diff --git a/02-data/02-size-1K/05-basic-transaction.parquet b/02-data/02-size-1K/05-basic-transaction.parquet
diff --git a/02-data/02-size-1K/05-basic_transaction.csv.gz b/02-data/02-size-1K/05-basic_transaction.csv.gz
diff --git a/02-data/02-size-1K/05-basic_transaction.parquet b/02-data/02-size-1K/05-basic_transaction.parquet
diff --git a/02-data/02-size-1K/06-basic-event.csv.gz b/02-data/02-size-1K/06-basic-event.csv.gz
diff --git a/02-data/02-size-1K/06-basic-event.parquet b/02-data/02-size-1K/06-basic-event.parquet
diff --git a/02-data/02-size-1K/06-basic_event.csv.gz b/02-data/02-size-1K/06-basic_event.csv.gz
diff --git a/02-data/02-size-1K/06-basic_event.parquet b/02-data/02-size-1K/06-basic_event.parquet
diff --git a/02-data/02-size-1K/07-basic-communication.csv.gz b/02-data/02-size-1K/07-basic-communication.csv.gz
diff --git a/02-data/02-size-1K/07-basic-communication.parquet b/02-data/02-size-1K/07-basic-communication.parquet
diff --git a/02-data/02-size-1K/07-basic_communication.csv.gz b/02-data/02-size-1K/07-basic_communication.csv.gz
diff --git a/02-data/02-size-1K/07-basic_communication.parquet b/02-data/02-size-1K/07-basic_communication.parquet
diff --git a/03-test/01-size-100.json b/03-test/01-size-100.json
diff --git a/03-test/02-size-1k.json b/03-test/02-size-1k.json
diff --git a/generator/base_data.py b/generator/base_data.py
@@ -81,7 +81,7 @@ def save(self, path, dir: str, compress: bool):
         # free memory
         del df
 
-    def apply_none_value(self, current_collection, property_name, default_value, lower_probability=1, none_value=""):
+    def apply_none_value(self, current_collection, property_name, default_value, lower_probability=1, none_value=None):
         """Apply None value, in case that current value is default.
          It is based on project setting (see setting in file 'model.json',
          with config values 'NONE_VALUES' and 'NONE_VALUES_PROBABILITY')

diff --git a/requirements.txt b/requirements.txt
@@ -3,7 +3,7 @@ click~=8.1
 
 numpy~=1.26
 pandas==2.2.2
-pyarrow==15.0.2
+pyarrow==16.1.0
 
 # generation of synthetic data
 faker~=24.2
diff --git a/tests/test_generator.py b/tests/test_generator.py
@@ -67,7 +67,7 @@ def test_generate(self):
     def test_generate_smallbulk_repeat(self):
         """Repeat generation of small files"""
 
-        for i in range(10):
+        for i in range(20):
             lbl = f"0-size-iter{i}-8,6"
 
             generator = SyntheticData(os.path.join("..","01-model"),TestGenerator.OUTPUT_ADR, TestGenerator.OUTPUT_ADR)
@@ -107,6 +107,18 @@ def test_generate_bigbulk(self):
         self.assertTrue(os.path.exists(dir))
         self.assertTrue(os.path.exists(path.join(dir, f"{basic_party.BasicParty.NAME}.csv")))
 
+    def test_generate_bigbulk_repeat(self):
+        for i in range(5):
+            lbl = f"0-size-iter{i}-1000,1000"
+
+            generator = SyntheticData(os.path.join("..","01-model"),TestGenerator.OUTPUT_ADR, TestGenerator.OUTPUT_ADR)
+            generator.generate(label=lbl, count=1000, bulk_max=1000, compress=False)
+
+            dir = path.join(TestGenerator.OUTPUT_ADR, lbl)
+            self.assertTrue(os.path.exists(dir))
+            self.assertTrue(os.path.exists(path.join(dir, f"{basic_party.BasicParty.NAME}.csv")))
+
+
     def _check_csv_header(self, filename, key_text):
         if os.path.exists(filename):
             df = pd.read_csv(filename)