Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ISSUE #34755] do not propagate parameters on InlineSchemaLoader #34853

Merged
merged 4 commits into from
Feb 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,11 @@ def propagate_types_and_parameters(
propagated_component["type"] = found_type

# When there is no resolved type, we're not processing a component (likely a regular object) and don't need to propagate parameters
if "type" not in propagated_component:
# When the type refers to a json schema, we're not processing a component as well. This check is currently imperfect as there could
# be json_schema are not objects but we believe this is not likely in our case because:
# * records are Mapping so objects hence SchemaLoader root should be an object
# * connection_specification is a Mapping
if "type" not in propagated_component or self._is_json_schema_object(propagated_component):
return propagated_component

# Combines parameters defined at the current level with parameters from parent components. Parameters at the current
Expand Down Expand Up @@ -140,3 +144,7 @@ def propagate_types_and_parameters(
if current_parameters:
propagated_component[PARAMETERS_STR] = current_parameters
return propagated_component

@staticmethod
def _is_json_schema_object(propagated_component: Mapping[str, Any]) -> bool:
return propagated_component.get("type") == "object"
Original file line number Diff line number Diff line change
Expand Up @@ -347,3 +347,60 @@ def test_only_propagate_parameters_to_components():
actual_component = transformer.propagate_types_and_parameters("", component, {})

assert actual_component == expected_component


def test_do_not_propagate_parameters_on_json_schema_object():
component = {
"type": "DeclarativeStream",
"streams": [
{
"type": "DeclarativeStream",
"schema_loader": {
"type": "InlineSchemaLoader",
"schema": {
"type": "object",
"$schema": "http://json-schema.org/schema#",
"properties": {"id": {"type": "string"}},
},
},
"$parameters": {
"name": "roasters",
"primary_key": "id",
},
}
],
}

expected_component = {
"type": "DeclarativeStream",
"streams": [
{
"type": "DeclarativeStream",
"name": "roasters",
"primary_key": "id",
"schema_loader": {
"type": "InlineSchemaLoader",
"name": "roasters",
"primary_key": "id",
"schema": {
"type": "object",
"$schema": "http://json-schema.org/schema#",
"properties": {"id": {"type": "string"}},
},
"$parameters": {
"name": "roasters",
"primary_key": "id",
},
},
"$parameters": {
"name": "roasters",
"primary_key": "id",
},
}
],
}

transformer = ManifestComponentTransformer()
actual_component = transformer.propagate_types_and_parameters("", component, {})

assert actual_component == expected_component
Loading