diff --git a/singer/transform.py b/singer/transform.py index 82888c2..49dd00e 100644 --- a/singer/transform.py +++ b/singer/transform.py @@ -35,6 +35,16 @@ def unix_seconds_to_datetime(value): return strftime(datetime.datetime.fromtimestamp(int(value), datetime.timezone.utc)) +def breadcrumb_path(breadcrumb): + """ + Transform breadcrumb into familiar object dot-notation + """ + name = ".".join(breadcrumb) + name = name.replace('properties.', '') + name = name.replace('.items', '[]') + return name + + class SchemaMismatch(Exception): def __init__(self, errors): if not errors: @@ -101,21 +111,25 @@ def __enter__(self): def __exit__(self, *args): self.log_warning() - def filter_data_by_metadata(self, data, metadata): + def filter_data_by_metadata(self, data, metadata, parent=()): if isinstance(data, dict) and metadata: for field_name in list(data.keys()): - selected = singer.metadata.get(metadata, ('properties', field_name), 'selected') - inclusion = singer.metadata.get(metadata, ('properties', field_name), 'inclusion') + breadcrumb = parent + ('properties', field_name) + selected = singer.metadata.get(metadata, breadcrumb, 'selected') + inclusion = singer.metadata.get(metadata, breadcrumb, 'inclusion') if inclusion == 'automatic': continue - if selected is False: + if (selected is False) or (inclusion == 'unsupported'): data.pop(field_name, None) - self.filtered.add(field_name) + self.filtered.add(breadcrumb_path(breadcrumb)) + else: + data[field_name] = self.filter_data_by_metadata( + data[field_name], metadata, breadcrumb) - if inclusion == 'unsupported': - data.pop(field_name, None) - self.filtered.add(field_name) + if isinstance(data, list) and metadata: + breadcrumb = parent + ('items',) + data = [self.filter_data_by_metadata(d, metadata, parent + ('items', )) for d in data] return data