Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restore original schema-ddl behaviour for objects with no defined fields #1369

Closed
wants to merge 1 commit into from
Closed

Restore original schema-ddl behaviour for objects with no defined fields #1369

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions ...wanalytics/snowplow/rdbloader/common/transformation/parquet/NonAtomicFieldsProvider.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import cats.syntax.all._
import com.snowplowanalytics.iglu.client.Resolver
import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup
import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema
import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{ArrayProperty, CommonProperties, ObjectProperty}
import com.snowplowanalytics.iglu.schemaddl.parquet.{Field, Type}
import com.snowplowanalytics.iglu.schemaddl.parquet.Migrations.mergeSchemas
import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent
Expand Down Expand Up @@ -125,6 +126,9 @@ object NonAtomicFieldsProvider {
.fetchSchemasWithSameModel[F](resolver, typeSet.max.schemaKey)
// Preserve the historic behaviour by dropping the schemas newer then max in this batch
.map(listOfSchemas => listOfSchemas.filter(_.schemaKey <= typeSet.max.schemaKey))
.map(_.map { case SchemaWithKey(key, schema) =>
SchemaWithKey(key, setAdditionalPropertiesTrue(schema))
})
.flatMap(schemaList =>
EitherT
.fromOption[F][LoaderIgluError, NonEmptyList[SchemaWithKey]](
Expand All @@ -139,6 +143,52 @@ object NonAtomicFieldsProvider {
)
)

/**
* Restores legacy behaviour of schema-ddl for backwards compatibility
*
* This concerns schemas like: `{ "type": "object", "additionalProperties": false }`
*
* - Older versions of schema-ddl converted this to a string (JSON) column.
* - Newer versions of schema-ddl convert this to a `None`, i.e. do not create a column for this
* schema.
*
* Here we convert the `additionalProperties` to `true` which tricks schema-ddl to return to the
* original behaviour.
*/
private def setAdditionalPropertiesTrue(schema: Schema): Schema = {
val items = schema.items.map {
case ArrayProperty.Items.ListItems(li) =>
ArrayProperty.Items.ListItems(setAdditionalPropertiesTrue(li))
case ArrayProperty.Items.TupleItems(ti) =>
ArrayProperty.Items.TupleItems(ti.map(setAdditionalPropertiesTrue))
}
val additionalItems = schema.additionalItems.map {
case ArrayProperty.AdditionalItems.AdditionalItemsAllowed(ail) =>
ArrayProperty.AdditionalItems.AdditionalItemsAllowed(ail)
case ArrayProperty.AdditionalItems.AdditionalItemsSchema(ais) =>
ArrayProperty.AdditionalItems.AdditionalItemsSchema(setAdditionalPropertiesTrue(ais))
}
val properties = schema.properties
.map { properties =>
properties.value.map { case (k, v) =>
k -> setAdditionalPropertiesTrue(v)
}
}
.map(ObjectProperty.Properties(_))

val additionalProperties =
if (schema.additionalProperties.isDefined) Some(ObjectProperty.AdditionalProperties.AdditionalPropertiesAllowed(false)) else None

schema.copy(
items = items,
properties = properties,
additionalProperties = additionalProperties,
additionalItems = additionalItems,
oneOf = schema.oneOf.map(oneOf => CommonProperties.OneOf(oneOf.value.map(setAdditionalPropertiesTrue))),
anyOf = schema.anyOf.map(anyOf => CommonProperties.AnyOf(anyOf.value.map(setAdditionalPropertiesTrue)))
)
}

private def fieldFromSchema(`type`: WideRow.Type)(schema: Schema): Option[Field] = {
val fieldName = SnowplowEvent.transformSchema(`type`.snowplowEntity.toSdkProperty, `type`.schemaKey)

Expand Down