From d0de0fdaa94ee030ca373d0a6436da50a1ab9edd Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Thu, 1 Sep 2022 15:15:11 -0700 Subject: [PATCH 01/12] Add test on the catalog diff logic --- .../io/airbyte/protocol/models/CatalogHelpersTest.java | 10 ++++++++++ .../src/test/resources/companies_schema.json | 1 + 2 files changed, 11 insertions(+) create mode 100644 airbyte-protocol/protocol-models/src/test/resources/companies_schema.json diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index a7777fa1ffe5..a3c8d75e0ab8 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -18,7 +18,9 @@ import java.util.Comparator; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; class CatalogHelpersTest { @@ -151,4 +153,12 @@ void testExtractIncrementalStreamDescriptors() { assertEquals("one", streamDescriptors.get(0).getName()); } + @Test + void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { + CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( + Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( + Collectors.toMap(Pair::getLeft, Pair::getRight)); + } + } + } diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json new file mode 100644 index 000000000000..93ddc120c93e --- /dev/null +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json @@ -0,0 +1 @@ +{"type": "object", "properties": {"id": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "plan": {"type": ["null", "object"], "properties": {"id": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}, "additionalProperties": false}, "size": {"type": ["null", "integer"]}, "tags": {"type": "object", "properties": {"tags": {"type": "array", "items": {"type": ["null", "object"], "anyOf": [{"type": "object", "properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "type": {"type": "string"}}}]}}, "type": {"type": "string"}}}, "type": {"type": ["null", "string"]}, "app_id": {"type": ["null", "string"]}, "website": {"type": ["null", "string"]}, "industry": {"type": ["null", "string"]}, "segments": {"type": "object", "properties": {"type": {"type": "string"}, "segments": {"type": "array", "items": {"type": ["null", "object"], "anyOf": [{"type": "object", "properties": {"id": {"type": "string"}, "type": {"type": "string"}}}]}}}}, "company_id": {"type": ["null", "string"]}, "created_at": {"type": ["null", "integer"]}, "updated_at": {"type": ["null", "integer"]}, "user_count": {"type": ["null", "integer"]}, "monthly_spend": {"type": ["null", "number"], "multipleOf": 1e-08}, "session_count": {"type": ["null", "integer"]}, "custom_attributes": {"type": ["null", "object"], "additionalProperties": true}, "remote_created_at": {"type": ["null", "integer"]}}, "additionalProperties": false} \ No newline at end of file From 1e605cb48fcecef14076952f2c3c733377bddd34 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Fri, 2 Sep 2022 07:39:39 -0700 Subject: [PATCH 02/12] Simplify test case --- .../java/io/airbyte/protocol/models/CatalogHelpersTest.java | 2 -- .../protocol-models/src/test/resources/companies_schema.json | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index a3c8d75e0ab8..4aca40810e64 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -158,7 +158,5 @@ void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( Collectors.toMap(Pair::getLeft, Pair::getRight)); - } } - } diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json index 93ddc120c93e..75674dab60b2 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json @@ -1 +1 @@ -{"type": "object", "properties": {"id": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "plan": {"type": ["null", "object"], "properties": {"id": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}, "additionalProperties": false}, "size": {"type": ["null", "integer"]}, "tags": {"type": "object", "properties": {"tags": {"type": "array", "items": {"type": ["null", "object"], "anyOf": [{"type": "object", "properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "type": {"type": "string"}}}]}}, "type": {"type": "string"}}}, "type": {"type": ["null", "string"]}, "app_id": {"type": ["null", "string"]}, "website": {"type": ["null", "string"]}, "industry": {"type": ["null", "string"]}, "segments": {"type": "object", "properties": {"type": {"type": "string"}, "segments": {"type": "array", "items": {"type": ["null", "object"], "anyOf": [{"type": "object", "properties": {"id": {"type": "string"}, "type": {"type": "string"}}}]}}}}, "company_id": {"type": ["null", "string"]}, "created_at": {"type": ["null", "integer"]}, "updated_at": {"type": ["null", "integer"]}, "user_count": {"type": ["null", "integer"]}, "monthly_spend": {"type": ["null", "number"], "multipleOf": 1e-08}, "session_count": {"type": ["null", "integer"]}, "custom_attributes": {"type": ["null", "object"], "additionalProperties": true}, "remote_created_at": {"type": ["null", "integer"]}}, "additionalProperties": false} \ No newline at end of file +{"type": "object", "properties": {"tags": {"type": "object", "properties": {"tags": {"type": "array", "items": {"type": ["null", "object"], "anyOf": [{"type": "object", "properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "type": {"type": "string"}}}]}}, "type": {"type": "string"}}}} , "additionalProperties": false} \ No newline at end of file From 0f3ec3ae5de9a398cbe7c4b6b44953633dab9972 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Thu, 8 Sep 2022 16:27:36 -0700 Subject: [PATCH 03/12] Fix the bug --- .../protocol/models/CatalogHelpers.java | 23 ++++++++++--- .../protocol/models/CatalogHelpersTest.java | 15 ++++++--- .../src/test/resources/companies_schema.json | 33 ++++++++++++++++++- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index e781d8586ced..f291c506237b 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -213,7 +213,7 @@ public static Set getTopLevelFieldNames(final ConfiguredAirbyteStream st */ @VisibleForTesting protected static Set getAllFieldNames(final JsonNode jsonSchema) { - return getFullyQualifiedFieldNamesWithTypes(jsonSchema) + return getFullyQualifiedFieldNamesWithTypes(jsonSchema, false) .stream() .map(Pair::getLeft) // only need field name, not fully qualified name @@ -232,7 +232,7 @@ protected static Set getAllFieldNames(final JsonNode jsonSchema) { * preorder. */ @VisibleForTesting - protected static List, JsonNode>> getFullyQualifiedFieldNamesWithTypes(final JsonNode jsonSchema) { + protected static List, JsonNode>> getFullyQualifiedFieldNamesWithTypes(final JsonNode jsonSchema, final boolean withNamedOneOf) { // if this were ever a performance issue, it could be replaced with a trie. this seems unlikely // however. final Set> fieldNamesThatAreOneOfs = new HashSet<>(); @@ -244,7 +244,20 @@ protected static List, JsonNode>> getFullyQualifiedFieldNamesW .stream() // first node is the original object. .skip(1) - .filter(fieldWithSchema -> filterChildrenOfFoneOneOf(fieldWithSchema.getLeft(), fieldWithSchema.getRight(), fieldNamesThatAreOneOfs)) + .filter(fieldWithSchema -> filterChildrenOfFoneOneOf(fieldWithSchema.getLeft(), + fieldWithSchema.getRight(), + fieldNamesThatAreOneOfs)) + .map(fieldWithSchema -> { + if (withNamedOneOf && isOneOfField(fieldWithSchema.getRight())) { + final List newPath = new ArrayList<>(fieldWithSchema.getLeft()); + newPath.add("oneOf"); + return Pair.of( + newPath, + fieldWithSchema.getRight()); + } else { + return fieldWithSchema; + } + }) .toList(); } @@ -332,10 +345,10 @@ public static Set getCatalogDiff(final AirbyteCatalog oldCatalo private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld, final AirbyteStream streamNew) { final Set fieldTransforms = new HashSet<>(); - final Map, JsonNode> fieldNameToTypeOld = getFullyQualifiedFieldNamesWithTypes(streamOld.getJsonSchema()) + final Map, JsonNode> fieldNameToTypeOld = getFullyQualifiedFieldNamesWithTypes(streamOld.getJsonSchema(), true) .stream() .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); - final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema()) + final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema(), true) .stream() .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 4aca40810e64..5b448db64c33 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -20,9 +20,12 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.val; import org.apache.commons.lang3.tuple.Pair; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +@SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") class CatalogHelpersTest { // handy for debugging test only. @@ -112,7 +115,7 @@ void testGetCatalogDiff() throws IOException { StreamTransform.createRemoveStreamTransform(new StreamDescriptor().withName("accounts")), StreamTransform.createUpdateStreamTransform(new StreamDescriptor().withName("users"), new UpdateStreamTransform(Set.of( FieldTransform.createAddFieldTransform(List.of("COD"), schema2.get(PROPERTIES).get("COD")), - FieldTransform.createRemoveFieldTransform(List.of("something2"), schema1.get(PROPERTIES).get("something2")), + FieldTransform.createRemoveFieldTransform(List.of("something2", "oneOf"), schema1.get(PROPERTIES).get("something2")), FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get(PROPERTIES).get("HKD")), FieldTransform.createUpdateFieldTransform(List.of(CAD), new UpdateFieldSchemaTransform( schema1.get(PROPERTIES).get(CAD), @@ -129,7 +132,8 @@ void testGetCatalogDiff() throws IOException { schema2.get(PROPERTIES).get(SOME_ARRAY).get(ITEMS).get(PROPERTIES).get("newName")))))) .sorted(STREAM_TRANSFORM_COMPARATOR) .toList(); - assertEquals(expectedDiff, actualDiff.stream().sorted(STREAM_TRANSFORM_COMPARATOR).toList()); + + Assertions.assertThat(actualDiff).containsAll(expectedDiff); } @Test @@ -155,8 +159,11 @@ void testExtractIncrementalStreamDescriptors() { @Test void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { - CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( - Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( + val test = CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( + Jsons.deserialize(MoreResources.readResource("companies_schema.json")), true); + + test.stream().collect( Collectors.toMap(Pair::getLeft, Pair::getRight)); } + } diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json index 75674dab60b2..0093b890e7f3 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json @@ -1 +1,32 @@ -{"type": "object", "properties": {"tags": {"type": "object", "properties": {"tags": {"type": "array", "items": {"type": ["null", "object"], "anyOf": [{"type": "object", "properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "type": {"type": "string"}}}]}}, "type": {"type": "string"}}}} , "additionalProperties": false} \ No newline at end of file +{ + "type": "object", + "properties": { + "tags": { + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": { + "type": ["null", "object"], + "anyOf": [ + { + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + } + ] + } + } + } + } + }, + "additionalProperties": false +} From cb4160e50018b9c1caa64da4d87728990a19c80c Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Thu, 8 Sep 2022 16:43:01 -0700 Subject: [PATCH 04/12] Rm useless val --- .../io/airbyte/protocol/models/CatalogHelpersTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 5b448db64c33..0f84c002cb6f 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -20,7 +20,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.val; import org.apache.commons.lang3.tuple.Pair; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -159,10 +158,8 @@ void testExtractIncrementalStreamDescriptors() { @Test void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { - val test = CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( - Jsons.deserialize(MoreResources.readResource("companies_schema.json")), true); - - test.stream().collect( + CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( + Jsons.deserialize(MoreResources.readResource("companies_schema.json")), true).stream().collect( Collectors.toMap(Pair::getLeft, Pair::getRight)); } From 7779a43665b1ba324f8722742fde2c32755715ae Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Thu, 8 Sep 2022 16:44:18 -0700 Subject: [PATCH 05/12] Format --- .../java/io/airbyte/protocol/models/CatalogHelpersTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 0f84c002cb6f..3cd4db02558e 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -160,7 +160,7 @@ void testExtractIncrementalStreamDescriptors() { void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( Jsons.deserialize(MoreResources.readResource("companies_schema.json")), true).stream().collect( - Collectors.toMap(Pair::getLeft, Pair::getRight)); + Collectors.toMap(Pair::getLeft, Pair::getRight)); } } From f53548ba5a06e8b9d6daffd7bec7dee11409fa02 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 9 Sep 2022 10:00:10 -0700 Subject: [PATCH 06/12] Update test input --- .../protocol-models/src/test/resources/companies_schema.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json index 0093b890e7f3..429709edb72b 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json @@ -7,9 +7,9 @@ "tags": { "type": "array", "items": { - "type": ["null", "object"], "anyOf": [ { + "type": "object", "properties": { "id": { "type": "integer" @@ -21,7 +21,8 @@ "type": "string" } } - } + }, + {"type": "null"} ] } } From c2ce936574588df19f73818933774af4be7f6957 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 9 Sep 2022 10:48:10 -0700 Subject: [PATCH 07/12] revert map change --- .../protocol/models/CatalogHelpers.java | 23 ++++--------------- .../protocol/models/CatalogHelpersTest.java | 2 +- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index f291c506237b..e781d8586ced 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -213,7 +213,7 @@ public static Set getTopLevelFieldNames(final ConfiguredAirbyteStream st */ @VisibleForTesting protected static Set getAllFieldNames(final JsonNode jsonSchema) { - return getFullyQualifiedFieldNamesWithTypes(jsonSchema, false) + return getFullyQualifiedFieldNamesWithTypes(jsonSchema) .stream() .map(Pair::getLeft) // only need field name, not fully qualified name @@ -232,7 +232,7 @@ protected static Set getAllFieldNames(final JsonNode jsonSchema) { * preorder. */ @VisibleForTesting - protected static List, JsonNode>> getFullyQualifiedFieldNamesWithTypes(final JsonNode jsonSchema, final boolean withNamedOneOf) { + protected static List, JsonNode>> getFullyQualifiedFieldNamesWithTypes(final JsonNode jsonSchema) { // if this were ever a performance issue, it could be replaced with a trie. this seems unlikely // however. final Set> fieldNamesThatAreOneOfs = new HashSet<>(); @@ -244,20 +244,7 @@ protected static List, JsonNode>> getFullyQualifiedFieldNamesW .stream() // first node is the original object. .skip(1) - .filter(fieldWithSchema -> filterChildrenOfFoneOneOf(fieldWithSchema.getLeft(), - fieldWithSchema.getRight(), - fieldNamesThatAreOneOfs)) - .map(fieldWithSchema -> { - if (withNamedOneOf && isOneOfField(fieldWithSchema.getRight())) { - final List newPath = new ArrayList<>(fieldWithSchema.getLeft()); - newPath.add("oneOf"); - return Pair.of( - newPath, - fieldWithSchema.getRight()); - } else { - return fieldWithSchema; - } - }) + .filter(fieldWithSchema -> filterChildrenOfFoneOneOf(fieldWithSchema.getLeft(), fieldWithSchema.getRight(), fieldNamesThatAreOneOfs)) .toList(); } @@ -345,10 +332,10 @@ public static Set getCatalogDiff(final AirbyteCatalog oldCatalo private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld, final AirbyteStream streamNew) { final Set fieldTransforms = new HashSet<>(); - final Map, JsonNode> fieldNameToTypeOld = getFullyQualifiedFieldNamesWithTypes(streamOld.getJsonSchema(), true) + final Map, JsonNode> fieldNameToTypeOld = getFullyQualifiedFieldNamesWithTypes(streamOld.getJsonSchema()) .stream() .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); - final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema(), true) + final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema()) .stream() .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 3cd4db02558e..13f5a3b4eb99 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -159,7 +159,7 @@ void testExtractIncrementalStreamDescriptors() { @Test void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( - Jsons.deserialize(MoreResources.readResource("companies_schema.json")), true).stream().collect( + Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( Collectors.toMap(Pair::getLeft, Pair::getRight)); } From ba5fd11a149b39976501544393227748ce138e9b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 9 Sep 2022 15:54:04 -0700 Subject: [PATCH 08/12] Add test --- .../protocol/models/CatalogHelpers.java | 34 ++++++++++- .../protocol/models/CatalogHelpersTest.java | 58 ++++++++++++++++++- .../resources/companies_schema_invalid.json | 34 +++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index e781d8586ced..0576b4604fa8 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -19,6 +19,7 @@ import io.airbyte.protocol.models.transform_models.UpdateStreamTransform; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -334,10 +335,16 @@ private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld final Set fieldTransforms = new HashSet<>(); final Map, JsonNode> fieldNameToTypeOld = getFullyQualifiedFieldNamesWithTypes(streamOld.getJsonSchema()) .stream() - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + .collect( + () -> new HashMap<>(), + CatalogHelpers::collectInHashMap, + CatalogHelpers::combineAccumulator + ); final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema()) .stream() - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + .collect(() -> new HashMap<>(), + CatalogHelpers::collectInHashMap, + CatalogHelpers::combineAccumulator); Sets.difference(fieldNameToTypeOld.keySet(), fieldNameToTypeNew.keySet()) .forEach(fieldName -> fieldTransforms.add(FieldTransform.createRemoveFieldTransform(fieldName, fieldNameToTypeOld.get(fieldName)))); @@ -354,4 +361,27 @@ private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld return new UpdateStreamTransform(fieldTransforms); } + @VisibleForTesting + static final JsonNode DUPLICATED_SCHEMA = Jsons.jsonNode("Duplicated Schema"); + + @VisibleForTesting + static void collectInHashMap(final HashMap, JsonNode> accumulator, final Pair, JsonNode> value) { + if (accumulator.containsKey(value.getKey())) { + accumulator.put(value.getKey(), DUPLICATED_SCHEMA); + } else { + accumulator.put(value.getKey(), value.getValue()); + } + } + + @VisibleForTesting + static void combineAccumulator(final HashMap, JsonNode> accumulatorLeft, final HashMap, JsonNode> accumulatorRight) { + accumulatorRight.forEach((key, value) -> { + if (accumulatorLeft.containsKey(key)) { + accumulatorLeft.put(key, DUPLICATED_SCHEMA); + } else { + accumulatorLeft.put(key, value); + } + }); + } + } diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 13f5a3b4eb99..dc73604e46ae 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -12,16 +12,19 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.protocol.models.transform_models.FieldTransform; import io.airbyte.protocol.models.transform_models.StreamTransform; +import io.airbyte.protocol.models.transform_models.StreamTransformType; import io.airbyte.protocol.models.transform_models.UpdateFieldSchemaTransform; import io.airbyte.protocol.models.transform_models.UpdateStreamTransform; import java.io.IOException; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; +import lombok.val; import org.assertj.core.api.Assertions; +import org.assertj.core.api.Condition; +import org.elasticsearch.common.collect.Map; import org.junit.jupiter.api.Test; @SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") @@ -160,7 +163,56 @@ void testExtractIncrementalStreamDescriptors() { void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( - Collectors.toMap(Pair::getLeft, Pair::getRight)); + () -> new HashMap<>(), + CatalogHelpers::collectInHashMap, + CatalogHelpers::combineAccumulator); } + @Test + void testGetFullyQualifiedFieldNamesWithTypesOnInvalidSchema() throws IOException { + val resultField = CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( + Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json"))).stream().collect( + () -> new HashMap<>(), + CatalogHelpers::collectInHashMap, + CatalogHelpers::combineAccumulator); + + Assertions.assertThat(resultField) + .contains( + Map.entry( + List.of("tags", "tags", "items"), + CatalogHelpers.DUPLICATED_SCHEMA + ) + ); + } + + @Test + void testGetCatalogDiffWithInvalidSchema() throws IOException { + final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json")); + final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource("companies_schema.json")); + final AirbyteCatalog catalog1 = new AirbyteCatalog().withStreams(List.of( + new AirbyteStream().withName("users").withJsonSchema(schema1))); + final AirbyteCatalog catalog2 = new AirbyteCatalog().withStreams(List.of( + new AirbyteStream().withName("users").withJsonSchema(schema2))); + + final Set actualDiff = CatalogHelpers.getCatalogDiff(catalog1, catalog2); + + Assertions.assertThat(actualDiff).hasSize(1); + Assertions.assertThat(actualDiff).first() + .has(new Condition(streamTransform -> streamTransform.getTransformType() == StreamTransformType.UPDATE_STREAM, + "Check update")); + } + + @Test + void testGetCatalogDiffWithBothInvalidSchema() throws IOException { + final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json")); + final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json")); + final AirbyteCatalog catalog1 = new AirbyteCatalog().withStreams(List.of( + new AirbyteStream().withName("users").withJsonSchema(schema1))); + final AirbyteCatalog catalog2 = new AirbyteCatalog().withStreams(List.of( + new AirbyteStream().withName("users").withJsonSchema(schema2))); + + final Set actualDiff = CatalogHelpers.getCatalogDiff(catalog1, catalog2); + + Assertions.assertThat(actualDiff).hasSize(0); + } } diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json new file mode 100644 index 000000000000..56af593f4382 --- /dev/null +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json @@ -0,0 +1,34 @@ +{ + "type": "object", + "properties": { + "tags": { + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": { + "type": ["null", "object"], + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + {"type": "null"} + ] + } + } + } + } + }, + "additionalProperties": false +} From 42b080007c9bcbb694debaa52a11c4a620ef64a3 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 9 Sep 2022 15:56:34 -0700 Subject: [PATCH 09/12] Fix test --- .../java/io/airbyte/protocol/models/CatalogHelpersTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index dc73604e46ae..5452bf6fc8b0 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -117,7 +117,7 @@ void testGetCatalogDiff() throws IOException { StreamTransform.createRemoveStreamTransform(new StreamDescriptor().withName("accounts")), StreamTransform.createUpdateStreamTransform(new StreamDescriptor().withName("users"), new UpdateStreamTransform(Set.of( FieldTransform.createAddFieldTransform(List.of("COD"), schema2.get(PROPERTIES).get("COD")), - FieldTransform.createRemoveFieldTransform(List.of("something2", "oneOf"), schema1.get(PROPERTIES).get("something2")), + FieldTransform.createRemoveFieldTransform(List.of("something2"), schema1.get(PROPERTIES).get("something2")), FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get(PROPERTIES).get("HKD")), FieldTransform.createUpdateFieldTransform(List.of(CAD), new UpdateFieldSchemaTransform( schema1.get(PROPERTIES).get(CAD), From ab3bfd87e0bea2b1e8154aed5de659001cb71e49 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 12 Sep 2022 09:47:12 -0700 Subject: [PATCH 10/12] Format --- .../airbyte/protocol/models/CatalogHelpers.java | 3 +-- .../protocol/models/CatalogHelpersTest.java | 17 ++++++++--------- .../src/test/resources/companies_schema.json | 2 +- .../resources/companies_schema_invalid.json | 2 +- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index 0576b4604fa8..63aa2cdfe81e 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -338,8 +338,7 @@ private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld .collect( () -> new HashMap<>(), CatalogHelpers::collectInHashMap, - CatalogHelpers::combineAccumulator - ); + CatalogHelpers::combineAccumulator); final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema()) .stream() .collect(() -> new HashMap<>(), diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 5452bf6fc8b0..ea14eae02d30 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -163,26 +163,24 @@ void testExtractIncrementalStreamDescriptors() { void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( - () -> new HashMap<>(), - CatalogHelpers::collectInHashMap, - CatalogHelpers::combineAccumulator); + () -> new HashMap<>(), + CatalogHelpers::collectInHashMap, + CatalogHelpers::combineAccumulator); } @Test void testGetFullyQualifiedFieldNamesWithTypesOnInvalidSchema() throws IOException { val resultField = CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json"))).stream().collect( - () -> new HashMap<>(), - CatalogHelpers::collectInHashMap, - CatalogHelpers::combineAccumulator); + () -> new HashMap<>(), + CatalogHelpers::collectInHashMap, + CatalogHelpers::combineAccumulator); Assertions.assertThat(resultField) .contains( Map.entry( List.of("tags", "tags", "items"), - CatalogHelpers.DUPLICATED_SCHEMA - ) - ); + CatalogHelpers.DUPLICATED_SCHEMA)); } @Test @@ -215,4 +213,5 @@ void testGetCatalogDiffWithBothInvalidSchema() throws IOException { Assertions.assertThat(actualDiff).hasSize(0); } + } diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json index 429709edb72b..014254081ccb 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema.json @@ -22,7 +22,7 @@ } } }, - {"type": "null"} + { "type": "null" } ] } } diff --git a/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json b/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json index 56af593f4382..00ab31012e9b 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json +++ b/airbyte-protocol/protocol-models/src/test/resources/companies_schema_invalid.json @@ -23,7 +23,7 @@ } } }, - {"type": "null"} + { "type": "null" } ] } } From 99e4d39db80ef3da389997ab2533a975c12f584f Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 12 Sep 2022 11:29:30 -0700 Subject: [PATCH 11/12] Use sugar --- .../main/java/io/airbyte/protocol/models/CatalogHelpers.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index 63aa2cdfe81e..23ba11b3d406 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -336,12 +336,13 @@ private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld final Map, JsonNode> fieldNameToTypeOld = getFullyQualifiedFieldNamesWithTypes(streamOld.getJsonSchema()) .stream() .collect( - () -> new HashMap<>(), + HashMap::new, CatalogHelpers::collectInHashMap, CatalogHelpers::combineAccumulator); final Map, JsonNode> fieldNameToTypeNew = getFullyQualifiedFieldNamesWithTypes(streamNew.getJsonSchema()) .stream() - .collect(() -> new HashMap<>(), + .collect( + HashMap::new, CatalogHelpers::collectInHashMap, CatalogHelpers::combineAccumulator); From d937b6fde4dbaf31377cd9075a4a445670a5124f Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 12 Sep 2022 14:32:23 -0700 Subject: [PATCH 12/12] Fix PMD errors --- .../protocol/models/CatalogHelpers.java | 4 +-- .../protocol/models/CatalogHelpersTest.java | 29 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index b2da9a8dca07..8c689db32012 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -365,7 +365,7 @@ private static UpdateStreamTransform getStreamDiff(final AirbyteStream streamOld static final JsonNode DUPLICATED_SCHEMA = Jsons.jsonNode("Duplicated Schema"); @VisibleForTesting - static void collectInHashMap(final HashMap, JsonNode> accumulator, final Pair, JsonNode> value) { + static void collectInHashMap(final Map, JsonNode> accumulator, final Pair, JsonNode> value) { if (accumulator.containsKey(value.getKey())) { accumulator.put(value.getKey(), DUPLICATED_SCHEMA); } else { @@ -374,7 +374,7 @@ static void collectInHashMap(final HashMap, JsonNode> accumulator, } @VisibleForTesting - static void combineAccumulator(final HashMap, JsonNode> accumulatorLeft, final HashMap, JsonNode> accumulatorRight) { + static void combineAccumulator(final Map, JsonNode> accumulatorLeft, final Map, JsonNode> accumulatorRight) { accumulatorRight.forEach((key, value) -> { if (accumulatorLeft.containsKey(key)) { accumulatorLeft.put(key, DUPLICATED_SCHEMA); diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index ea14eae02d30..7be314d1128a 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -37,6 +37,9 @@ class CatalogHelpersTest { private static final String ITEMS = "items"; private static final String SOME_ARRAY = "someArray"; private static final String PROPERTIES = "properties"; + private static final String USERS = "users"; + private static final String COMPANIES_VALID = "companies_schema.json"; + private static final String COMPANIES_INVALID = "companies_schema_invalid.json"; @Test void testFieldToJsonSchema() { @@ -105,17 +108,17 @@ void testGetCatalogDiff() throws IOException { final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource("valid_schema.json")); final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource("valid_schema2.json")); final AirbyteCatalog catalog1 = new AirbyteCatalog().withStreams(List.of( - new AirbyteStream().withName("users").withJsonSchema(schema1), + new AirbyteStream().withName(USERS).withJsonSchema(schema1), new AirbyteStream().withName("accounts").withJsonSchema(Jsons.emptyObject()))); final AirbyteCatalog catalog2 = new AirbyteCatalog().withStreams(List.of( - new AirbyteStream().withName("users").withJsonSchema(schema2), + new AirbyteStream().withName(USERS).withJsonSchema(schema2), new AirbyteStream().withName("sales").withJsonSchema(Jsons.emptyObject()))); final Set actualDiff = CatalogHelpers.getCatalogDiff(catalog1, catalog2); final List expectedDiff = Stream.of( StreamTransform.createAddStreamTransform(new StreamDescriptor().withName("sales")), StreamTransform.createRemoveStreamTransform(new StreamDescriptor().withName("accounts")), - StreamTransform.createUpdateStreamTransform(new StreamDescriptor().withName("users"), new UpdateStreamTransform(Set.of( + StreamTransform.createUpdateStreamTransform(new StreamDescriptor().withName(USERS), new UpdateStreamTransform(Set.of( FieldTransform.createAddFieldTransform(List.of("COD"), schema2.get(PROPERTIES).get("COD")), FieldTransform.createRemoveFieldTransform(List.of("something2"), schema1.get(PROPERTIES).get("something2")), FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get(PROPERTIES).get("HKD")), @@ -162,7 +165,7 @@ void testExtractIncrementalStreamDescriptors() { @Test void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( - Jsons.deserialize(MoreResources.readResource("companies_schema.json"))).stream().collect( + Jsons.deserialize(MoreResources.readResource(COMPANIES_VALID))).stream().collect( () -> new HashMap<>(), CatalogHelpers::collectInHashMap, CatalogHelpers::combineAccumulator); @@ -171,7 +174,7 @@ void testGetFullyQualifiedFieldNamesWithTypes() throws IOException { @Test void testGetFullyQualifiedFieldNamesWithTypesOnInvalidSchema() throws IOException { val resultField = CatalogHelpers.getFullyQualifiedFieldNamesWithTypes( - Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json"))).stream().collect( + Jsons.deserialize(MoreResources.readResource(COMPANIES_INVALID))).stream().collect( () -> new HashMap<>(), CatalogHelpers::collectInHashMap, CatalogHelpers::combineAccumulator); @@ -185,12 +188,12 @@ void testGetFullyQualifiedFieldNamesWithTypesOnInvalidSchema() throws IOExceptio @Test void testGetCatalogDiffWithInvalidSchema() throws IOException { - final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json")); - final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource("companies_schema.json")); + final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource(COMPANIES_INVALID)); + final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource(COMPANIES_VALID)); final AirbyteCatalog catalog1 = new AirbyteCatalog().withStreams(List.of( - new AirbyteStream().withName("users").withJsonSchema(schema1))); + new AirbyteStream().withName(USERS).withJsonSchema(schema1))); final AirbyteCatalog catalog2 = new AirbyteCatalog().withStreams(List.of( - new AirbyteStream().withName("users").withJsonSchema(schema2))); + new AirbyteStream().withName(USERS).withJsonSchema(schema2))); final Set actualDiff = CatalogHelpers.getCatalogDiff(catalog1, catalog2); @@ -202,12 +205,12 @@ void testGetCatalogDiffWithInvalidSchema() throws IOException { @Test void testGetCatalogDiffWithBothInvalidSchema() throws IOException { - final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json")); - final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource("companies_schema_invalid.json")); + final JsonNode schema1 = Jsons.deserialize(MoreResources.readResource(COMPANIES_INVALID)); + final JsonNode schema2 = Jsons.deserialize(MoreResources.readResource(COMPANIES_INVALID)); final AirbyteCatalog catalog1 = new AirbyteCatalog().withStreams(List.of( - new AirbyteStream().withName("users").withJsonSchema(schema1))); + new AirbyteStream().withName(USERS).withJsonSchema(schema1))); final AirbyteCatalog catalog2 = new AirbyteCatalog().withStreams(List.of( - new AirbyteStream().withName("users").withJsonSchema(schema2))); + new AirbyteStream().withName(USERS).withJsonSchema(schema2))); final Set actualDiff = CatalogHelpers.getCatalogDiff(catalog1, catalog2);