fix mapper and tests

open-contracting · Oct 8, 2024 · 0fe1e16 · 0fe1e16
1 parent 4e19fef
commit 0fe1e16
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 5 deletions.
diff --git a/nightingale/mapper.py b/nightingale/mapper.py
@@ -205,14 +205,16 @@ def set_nested_value(nested_dict, keys, value, schema, add_new=False, append_onc
                         # case for /parties/roles
                         set_nested_value(result, keys, value, flattened_schema, add_new=True, append_once=True)
                         continue
+                    elif array_counters is None:
+                        array_counters = {}
                     elif array_path in array_counters:
                         if add_new := is_new_array(array_counters, child_path, last_key_name, array_value, array_path):
                             array_counters[array_path] = array_value
                             set_nested_value(result, keys[:-1], {}, flattened_schema, add_new=add_new)
                     else:
                         if last_key_name == "id":
                             array_counters[array_path] = array_value
-                        set_nested_value(result, keys[:-1], [{}], flattened_schema)
+                            set_nested_value(result, keys[:-1], {}, flattened_schema, True)
 
                     current = result
                     for i, key in enumerate(keys[:-1]):
@@ -236,7 +238,7 @@ def set_nested_value(nested_dict, keys, value, schema, add_new=False, append_onc
     def shift_current_array(self, current, array_path, array_counters):
         if not current:
             current.append({})
-        return find_array_element_by_id(current, array_counters.get(array_path))
+        return find_array_element_by_id(current, array_counters.get(array_path) if array_counters else None)
 
     def make_release_id(self, curr_row: dict) -> None:
         """

diff --git a/tests/test_loader.py b/tests/test_loader.py
@@ -33,7 +33,7 @@ def test_get_connection(self):
 
     def test_load(self):
         data = self.loader.load("SELECT * FROM test_table")
-        self.assertEqual(data, [{"column1": "value1"}])
+        self.assertEqual(list(data), [{"column1": "value1"}])
 
 
 if __name__ == "__main__":

diff --git a/tests/test_publisher.py b/tests/test_publisher.py
@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 
 from nightingale.config import Publishing
 from nightingale.publisher import DataPublisher
@@ -7,7 +8,9 @@
 class TestDataPublisher(unittest.TestCase):
     def setUp(self):
         self.config = Publishing(**{"version": "1.0", "publisher": "test_publisher", "base_uri": "http://example.com"})
-        self.publisher = DataPublisher(self.config)
+        self.mapping = mock.Mock()
+        self.mapping.extensions = [{"url": "http://example.com/extension1"}, {"url": None}]
+        self.publisher = DataPublisher(self.config, self.mapping)
         self.data = [{"ocid": "ocid_prefix-1234"}]
 
     def test_publish(self):

diff --git a/tests/test_unflatten.py b/tests/test_unflatten.py
@@ -25,6 +25,10 @@ def get_ocid_mapping(self):
     def get_containing_array_path(self, path):
         return get_longest_array_path(self.array_paths, path)
 
+@pytest.fixture(autouse=True)
+def mock_load_workbook():
+    with mock.patch("openpyxl.load_workbook", return_value=mock.MagicMock()):
+        yield
 
 @pytest.fixture
 def mock_config():
@@ -54,12 +58,12 @@ def mock_config():
             MockMappingConfig(
                 {
                     "flat_col1": "/object/field1",
-                    "flat_col_s": "/object/field2/string_field",
                     "flat_col2": "/object/field2/array_field/id",
                     "flat_col3": "/object/field2/array_field/array_field2/id",
                     "flat_col4": "/object/field2/array_field/array_field2/id",
                     "flat_col5": "/object/field2/array_field/id",
                     "flat_col6": "/object/field2/array_field/array_field2/id",
+                    "flat_col_s": "/object/field2/string_field",
                 },
                 array_paths=["/object/field2/array_field", "/object/field2/array_field/array_field2"],
             ),
@@ -289,6 +293,65 @@ def mock_config():
             None,
             {"root": {"field1": "1", "array1": [{"id": "2", "field2": "3"}, {"id": "4"}]}},
         ),
+        (
+            {
+                "col1": "val1",
+                "col2": "val2",
+                "col3": "val3",
+                "col4": "val4",
+                "col5": "val5",
+                "col6": "val6",
+                "col7": "val7"
+            },
+            MockMappingConfig(
+                {
+                    "col1": "/object/lvl1/field",
+                    "col2": "/object/lvl1/array1/field1",
+                    "col3": "/object/lvl1/array1/field2",
+                    "col4": "/object/lvl1/array1/id",
+                    "col5": "/object/lvl1/lvl2/field",
+                    "col6": "/object/lvl1/lvl2/array2/field1",
+                    "col7": "/object/lvl1/lvl2/array2/id",
+                },
+                array_paths=["/lvl1/array"],
+            ),
+            {
+                "/object/lvl1/field": {"type": "string"},
+                "/object/lvl1/array1": {"type": "array"},
+                "/object/lvl1/array1/field1": {"type": "string"},
+                "/object/lvl1/array1/field2": {"type": "string"},
+                "/object/lvl1/array1/id": {"type": "string"},
+                "/object/lvl1/lvl2/field": {"type": "string"},
+                "/object/lvl1/lvl2/array2": {"type": "array"},
+                "/object/lvl1/lvl2/array2/field1": {"type": "string"},
+                "/object/lvl1/lvl2/array2/id": {"type": "string"},
+            },
+            None,
+            {
+                "object": {
+                    "lvl1": {
+                        "field": "val1",
+                        "array1": [
+                            {
+                                "id": "val4",
+                                "field1": "val2",
+                                "field2": "val3"
+                            },
+                        ],
+                        "lvl2": {
+                            "field": "val5",
+                            "array2": [
+                                {
+                                    "id": "val7",
+                                    "field1": "val6"
+                                }
+                            ]
+                        }
+
+                    }
+                }
+            }
+        ),
     ],
 )
 def test_transform_data(input_data, mapping_config, flattened_schema, result, expected_output, mock_config):