diff --git a/.gitignore b/.gitignore index f086f1ab..a7befb81 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ rabbitmq/* .vscode/ staticfiles/ sample_data/downloads/* +jupyter/.ipynb_checkpoints # osmnx data cache folder cache diff --git a/dev/Dockerfile b/dev/Dockerfile index e2fa8216..d09d5bb3 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -15,7 +15,7 @@ RUN python -m pip install ./tile2net COPY ./setup.py /opt/uvdat-server/setup.py COPY ./manage.py /opt/uvdat-server/manage.py COPY ./uvdat /opt/uvdat-server/uvdat -RUN pip install large-image[gdal,pil] large-image-converter --find-links https://girder.github.io/large_image_wheels +RUN pip install large-image[gdal,pil,mapnik] large-image-converter --find-links https://girder.github.io/large_image_wheels RUN pip install --editable /opt/uvdat-server[dev] # Use a directory name which will never be an import name, as isort considers this as first-party. diff --git a/jupyter/requirements.txt b/jupyter/requirements.txt new file mode 100644 index 00000000..c84a9168 --- /dev/null +++ b/jupyter/requirements.txt @@ -0,0 +1,3 @@ +ipyleaflet +ipywidgets +ipytree diff --git a/jupyter/uvdat_data_exploration.ipynb b/jupyter/uvdat_data_exploration.ipynb new file mode 100644 index 00000000..ee315365 --- /dev/null +++ b/jupyter/uvdat_data_exploration.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d3435282-d633-4edf-8131-462a038306f1", + "metadata": {}, + "outputs": [], + "source": [ + "from uvdat_explorer import UVDATExplorer\n", + "\n", + "UVDATExplorer(\n", + " api_url='http://localhost:8000/api/v1',\n", + " # email='myemail',\n", + " # password='mypassword',\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/uvdat_explorer.py b/jupyter/uvdat_explorer.py new file mode 100644 index 00000000..11169a43 --- /dev/null +++ b/jupyter/uvdat_explorer.py @@ -0,0 +1,258 @@ +from urllib.parse import urlencode + +from IPython import display +from ipyleaflet import FullScreenControl, Map, TileLayer, VectorTileLayer, basemaps, projections +from ipytree import Node, Tree +import ipywidgets as widgets +import requests + +DEFAULT_CENTER = [42.36, -71.06] +DEFAULT_ZOOM = 14 + + +class LayerRepresentation: + def __init__(self, layer, api_url, session, token, center, zoom): + self.layer = layer + self.session = session + self.api_url = api_url + self.token = token + self.center = center + self.zoom = zoom + + self.output = widgets.Output() + self.frame_index = 0 + self.frames = self.layer.get('frames', []) + self.max_frame = max(frame.get('index') for frame in self.frames) if len(self.frames) else 0 + self.play_widget = widgets.Play( + min=0, + max=self.max_frame, + interval=1500, + ) + self.frame_slider = widgets.IntSlider( + description='Frame Index:', + min=0, + max=self.max_frame, + ) + widgets.jslink((self.play_widget, 'value'), (self.frame_slider, 'value')) + label_text = 'Frame Name: ' + if len(self.frames): + label_text += self.frames[0].get('name') + self.frame_name_label = widgets.Label(label_text) + self.frame_slider.observe(self.update_frame) + self.map = Map( + crs=projections.EPSG3857, + basemap=basemaps.OpenStreetMap.Mapnik, + center=self.center, + zoom=self.zoom, + max_zoom=20, + min_zoom=0, + scroll_wheel_zoom=True, + dragging=True, + attribution_control=False, + ) + self.map.add(FullScreenControl()) + self.map_layers = [] + self.update_frame(dict(name='value')) + + def get_frame_path_and_metadata(self, frame): + raster = frame.get('raster') + vector = frame.get('vector') + path, metadata = None, None + if raster: + raster_id = raster.get('id') + path = f'rasters/{raster_id}/' + metadata = raster.get('metadata') + elif vector: + vector_id = vector.get('id') + path = f'vectors/{vector_id}/' + metadata = vector.get('metadata') + return path, metadata + + def get_flat_filters(self, filters): + flat = {} + for key, value in filters.items(): + if isinstance(value, dict): + for k, v in self.get_flat_filters(value).items(): + flat[f'{key}.{k}'] = v + else: + flat[key] = value + return flat + + def update_frame(self, event): + with self.output: + if event.get('name') == 'value': + for map_layer in self.map_layers: + self.map.remove_layer(map_layer) + self.map_layers = [] + + self.frame_index = int(event.get('new', 0)) + current_frames = [ + frame for frame in self.frames if frame.get('index') == self.frame_index + ] + for frame in current_frames: + tile_size = 256 + frame_name = frame.get('name') + self.frame_name_label.value = f'Frame Name: {frame_name}' + url_path, metadata = self.get_frame_path_and_metadata(frame) + if metadata is not None: + tile_size = metadata.get('tileWidth', 256) + url_suffix = 'tiles/{z}/{x}/{y}' + layer_class = None + layer_kwargs = dict(min_zoom=0, max_zoom=20, tile_size=tile_size) + query = dict(token=self.token) + source_filters = frame.get('source_filters') + if source_filters is not None and source_filters != dict(band=1): + query.update(self.get_flat_filters(source_filters)) + + if 'raster' in url_path: + url_suffix += '.png' + layer_class = TileLayer + query.update(projection='EPSG:3857') + elif 'vector' in url_path: + layer_class = VectorTileLayer + if layer_class is not None: + query_string = urlencode(query) + map_layer = layer_class( + url=self.api_url + url_path + url_suffix + '?' + query_string, + **layer_kwargs, + ) + self.map_layers.append(map_layer) + self.map.add_layer(map_layer) + + def get_widget(self): + children = [ + self.map, + self.output, + ] + if self.max_frame: + children = [self.frame_slider, self.play_widget, self.frame_name_label, *children] + return widgets.VBox(children) + + +class UVDATExplorer: + def __init__(self, api_url=None, email=None, password=None, center=None, zoom=None): + if api_url is None: + msg = 'UVDATExplorer missing argument: %s must be specified.' + raise ValueError(msg % '`api_url`') + if not api_url.endswith('/'): + api_url += '/' + self.api_url = api_url + self.session = requests.Session() + self.token = None + self.authenticated = False + self.email = email + self.password = password + self.center = center or DEFAULT_CENTER + self.zoom = zoom or DEFAULT_ZOOM + + # Widgets + self.tree = None + self.tree_nodes = {} + self.output = widgets.Output() + self.email_input = widgets.Text(description='Email:') + self.password_input = widgets.Password(description='Password:') + self.button = widgets.Button(description='Get Datasets') + self.button.on_click(self.get_datasets) + children = [self.output] + + if email is None: + children.append(self.email_input) + if password is None: + children.append(self.password_input) + + if email and password: + authenticated = self.authenticate() + if authenticated: + children.append(widgets.Label('Session Authenticated.')) + children.append(self.button) + + # Display + self.display = display.display(widgets.VBox(children), display_id=True) + self.update_display(children) + + def __del__(self): + self.session.close() + + def authenticate(self): + with self.output: + self.output.clear_output() + email = self.email or self.email_input.value + password = self.password or self.password_input.value + self.email_input.value = '' + self.password_input.value = '' + + response = requests.post( + self.api_url + 'token/', + dict( + username=email, + password=password, + ), + ) + if response.status_code == 200: + self.token = response.json().get('token') + self.session.headers['Authorization'] = f'Token {self.token}' + self.authenticated = True + return True + else: + print('Invalid login.') + return False + + def get_datasets(self, *args): + with self.output: + if not self.authenticated: + self.authenticate() + response = self.session.get(self.api_url + 'datasets') + response.raise_for_status() + datasets = response.json().get('results') + + self.tree = Tree() + for dataset in datasets: + node = Node(dataset.get('name'), icon='database') + node.observe(self.get_dataset_layers, 'selected') + self.tree_nodes[node._id] = dataset + self.tree.add_node(node) + + children = [self.tree, self.output] + self.update_display(children) + + def get_dataset_layers(self, event): + with self.output: + node = event.get('owner') + for child in node.nodes: + node.remove_node(child) + node_id = node._id + dataset = self.tree_nodes[node_id] + dataset_id = dataset.get('id') + + response = self.session.get(self.api_url + f'datasets/{dataset_id}/layers') + response.raise_for_status() + layers = response.json() + + for layer in layers: + child_node = Node(layer.get('name'), icon='file') + child_node.observe(self.select_layer, 'selected') + self.tree_nodes[child_node._id] = layer + node.add_node(child_node) + + def select_layer(self, event): + with self.output: + node = event.get('owner') + node_id = node._id + layer = self.tree_nodes[node_id] + + self.map = LayerRepresentation( + layer, + self.api_url, + self.session, + self.token, + self.center, + self.zoom, + ) + children = [self.tree, self.output, self.map.get_widget()] + self.update_display(children) + + def update_display(self, children): + self.display.update(widgets.VBox(children)) + + def _ipython_display_(self): + return self.display diff --git a/sample_data/ingest_use_case.py b/sample_data/ingest_use_case.py index 68b9a94f..dc3c4e20 100644 --- a/sample_data/ingest_use_case.py +++ b/sample_data/ingest_use_case.py @@ -28,16 +28,16 @@ def ingest_file(file_info, index=0, dataset=None, chart=None): file_location = Path(DOWNLOADS_FOLDER, file_path) file_type = file_path.split('.')[-1] if not file_location.exists(): - print(f'\t Downloading data file {file_name}.') + print(f'\t\t Downloading data file {file_name}.') file_location.parent.mkdir(parents=True, exist_ok=True) with open(file_location, 'wb') as f: r = requests.get(file_url) r.raise_for_status() f.write(r.content) - existing = FileItem.objects.filter(name=file_name) + existing = FileItem.objects.filter(dataset=dataset, name=file_name) if existing.count(): - print('\t', f'FileItem {file_name} already exists.') + print('\t\t', f'FileItem {file_name} already exists.') else: new_file_item = FileItem.objects.create( name=file_name, @@ -51,7 +51,7 @@ def ingest_file(file_info, index=0, dataset=None, chart=None): ), index=index, ) - print('\t', f'FileItem {new_file_item.name} created.') + print('\t\t', f'FileItem {new_file_item.name} created.') with file_location.open('rb') as f: new_file_item.file.save(file_path, ContentFile(f.read())) @@ -74,7 +74,7 @@ def ingest_projects(use_case): }, ) if created: - print('\t', f'Project {project_for_setting.name} created.') + print('\t\t', f'Project {project_for_setting.name} created.') project_for_setting.datasets.set(Dataset.objects.filter(name__in=project['datasets'])) project_for_setting.set_permissions(owner=User.objects.filter(is_superuser=True).first()) @@ -100,7 +100,7 @@ def ingest_charts(use_case): metadata=chart.get('metadata'), editable=chart.get('editable', False), ) - print('\t', f'Chart {new_chart.name} created.') + print('\t\t', f'Chart {new_chart.name} created.') for index, file_info in enumerate(chart.get('files', [])): ingest_file( file_info, @@ -109,7 +109,7 @@ def ingest_charts(use_case): ) chart_for_conversion = new_chart - print('\t', f'Converting data for {chart_for_conversion.name}...') + print('\t\t', f'Converting data for {chart_for_conversion.name}.') chart_for_conversion.spawn_conversion_task( conversion_options=chart.get('conversion_options'), asynchronous=False, @@ -124,6 +124,7 @@ def ingest_datasets(use_case, include_large=False, dataset_indexes=None): data = json.load(datasets_json) for index, dataset in enumerate(data): if dataset_indexes is None or index in dataset_indexes: + print('\t- ', dataset['name']) existing = Dataset.objects.filter(name=dataset['name']) if existing.count(): dataset_for_conversion = existing.first() @@ -133,10 +134,8 @@ def ingest_datasets(use_case, include_large=False, dataset_indexes=None): name=dataset['name'], description=dataset['description'], category=dataset['category'], - dataset_type=dataset.get('type', 'vector').upper(), metadata=dataset.get('metadata', {}), ) - print('\t', f'Dataset {new_dataset.name} created.') for index, file_info in enumerate(dataset.get('files', [])): ingest_file( file_info, diff --git a/sample_data/use_cases/boston_floods/datasets.json b/sample_data/use_cases/boston_floods/datasets.json index b8baa520..7d46a205 100644 --- a/sample_data/use_cases/boston_floods/datasets.json +++ b/sample_data/use_cases/boston_floods/datasets.json @@ -3,120 +3,175 @@ "name": "MBTA Rapid Transit", "description": "Boston Subway System Lines and Stops", "category": "transportation", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64b95be1cf6a0eaef6c141f6/download", - "path": "boston/mbta_rapid_transit.zip" + "path": "boston/mbta_rapid_transit.zip", + "metadata": { + "combine_contents": "true" + } } ], "network_options": { "connection_column": "ROUTE", "connection_column_delimiter": "/", "node_id_column": "STATION" - }, - "style_options": { - "color_property": "LINE", - "color_delimiter": "/", - "outline": "white" } }, { "name": "MBTA Commuter Rail", "description": "Rail linework and station points for passenger, freight, and Amtrak and MBTA Commuter Rail trains.", "category": "transportation", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64907c77f04fb368544295ed/download", - "path": "boston/commuter_rail.zip" + "path": "boston/commuter_rail.zip", + "metadata": { + "combine_contents": "true" + } } - ], - "style_options": { - "color_property": "gray", - "outline": "black" - } + ] }, { "name": "Boston Hurricane Surge Inundation Zones", "description": "This layer represents worst-case Hurricane Surge Inundation areas for Category 1 through 4 hurricanes striking the coast of Massachusetts.", "category": "climate", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64907b6ef04fb368544295e7/download", "path": "boston/hurr_inun.zip" } - ], - "style_options": { - "color_property": "HURR_CAT", - "palette": { - "1": "powderblue", - "2": "deepskyblue", - "3": "royalblue", - "4": "mediumblue", - "5": "darkblue" - } - } + ] }, { "name": "Boston FEMA National Flood Hazard", "description": "The National Flood Hazard Layer (NFHL) dataset represents the current effective flood risk data for those parts of the country where maps have been modernized by the Federal Emergency Management Agency (FEMA).", "category": "climate", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64907beff04fb368544295ea/download", "path": "boston/flood_hazard_fema.zip" } - ], - "style_options": { - "color_property": "FLD_ZONE", - "palette": { - "A": "gold", - "AE": "orange", - "AH": "tomato", - "AO": "orangered", - "VE": "darkred", - "D": "gray", - "X": "green" - } - } + ] }, { "name": "Massachusetts Elevation Data", "description": "Low resolution sampling of land elevation in meters above sea level", "category": "elevation", - "type": "raster", "files": [ { "url": "https://data.kitware.com/api/v1/item/64907e4df04fb368544295f0/download", "path": "boston/easternmass.tif" } ], - "style_options": { - "transparency_threshold": 1, - "trim_distribution_percentage": 0.01 - } + "layers": [ + { + "name": "Elevation", + "frames": [ + { + "name": "Band 1", + "data": "easternmass.tif" + } + ] + } + ] }, { "name": "Boston Orthoimagery", "description": "Sourced from https://gis.data.mass.gov/maps/6c7009c789354573a42af7251fb768a4/about", "category": "imagery", - "type": "raster", "files": [ { "url": "https://data.kitware.com/api/v1/item/6744df3c22f196cb5d931161/download", "path": "boston/boston_orthoimagery.tiff" + }, + { + "url": "https://data.kitware.com/api/v1/item/677c69b537ed172242a867b1/download", + "path": "boston/orthoimagery_tiles.zip" } ], - "style_options": {} + "layers": [ + { + "name": "Orthoimagery from a single precomposited image", + "frames": [ + { + "name": "2023", + "index": 0, + "data": "boston_orthoimagery.tiff" + } + ] + }, + { + "name": "Orthoimagery from many images in a zip", + "frames": [ + { + "name": "2019", + "index": 0, + "data": "2019_19TCG285905.jp2" + }, + { + "name": "2019", + "index": 0, + "data": "2019_19TCG285920.jp2" + }, + { + "name": "2019", + "index": 0, + "data": "2019_19TCG300905.jp2" + }, + { + "name": "2019", + "index": 0, + "data": "2019_19TCG300920.jp2" + }, + { + "name": "2021", + "index": 1, + "data": "2021_19TCG285905.jp2" + }, + { + "name": "2021", + "index": 1, + "data": "2021_19TCG285920.jp2" + }, + { + "name": "2021", + "index": 1, + "data": "2021_19TCG300905.jp2" + }, + { + "name": "2021", + "index": 1, + "data": "2021_19TCG300920.jp2" + }, + { + "name": "2023", + "index": 2, + "data": "2023_19TCG285905.jp2" + }, + { + "name": "2023", + "index": 2, + "data": "2023_19TCG285920.jp2" + }, + { + "name": "2023", + "index": 2, + "data": "2023_19TCG300905.jp2" + }, + { + "name": "2023", + "index": 2, + "data": "2023_19TCG300920.jp2" + } + ] + } + ] }, { "name": "Boston Neighborhoods", "description": "The Neighborhood boundaries data layer is a combination of zoning neighborhood boundaries, zip code boundaries and 2010 Census tract boundaries", "category": "region", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64caa3da77edef4e1ea8ef57/download", @@ -125,19 +180,12 @@ ], "region_options": { "name_property": "BlockGr202" - }, - "style_options": { - "outline": "white", - "palette": [ - "blue" - ] } }, { "name": "Boston Census 2020 Block Groups", "description": "Block groups (between 600 and 3,000 people per block) for Boston from the 2020 census", "category": "region", - "type": "vector", "files": [ { "url": "https://data.boston.gov/dataset/c478b600-3e3e-46fd-9f57-da89459e9928/resource/11282722-9386-4272-8a82-2fcec89e6d55/download/census2020_blockgroups.zip", @@ -146,19 +194,12 @@ ], "region_options": { "name_property": "GEOID20" - }, - "style_options": { - "outline": "white", - "palette": [ - "green" - ] } }, { "name": "Boston Zip Codes", "description": "Zip codes 01001-02791", "category": "region", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64fbb4c6e99e9e6006f00114/download", @@ -167,19 +208,12 @@ ], "region_options": { "name_property": "GEOID20" - }, - "style_options": { - "outline": "white", - "palette": [ - "grey" - ] } }, { - "name": "Boston Sea Level Rises", - "description": "From Analyze Boston: flood data for 9-inch, 21-inch, and 36-inch sea-level rise projections", + "name": "Boston Projected Flood Events", + "description": "Flood projections from Analyze Boston", "category": "flood", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/64f8743d6725af35134c1479/download", @@ -192,21 +226,7 @@ { "url": "https://data.kitware.com/api/v1/item/64f874566725af35134c147f/download", "path": "boston/36in_rise.geojson" - } - ], - "style_options": { - "outline": "white", - "palette": [ - "blue" - ] - } - }, - { - "name": "Boston 10-Year Flood Events", - "description": "From Analyze Boston: flood data for 9-inch, 21-inch, and 36-inch 10-year flood projections", - "category": "flood", - "type": "vector", - "files": [ + }, { "url": "https://data.kitware.com/api/v1/item/64f87ba86725af35134c1485/download", "path": "boston/9in_10yr_flood.geojson" @@ -218,21 +238,7 @@ { "url": "https://data.kitware.com/api/v1/item/64f87bfd6725af35134c1491/download", "path": "boston/36in_10yr_flood.geojson" - } - ], - "style_options": { - "outline": "white", - "palette": [ - "blue" - ] - } - }, - { - "name": "Boston 100-Year Flood Events", - "description": "From Analyze Boston: flood data for 9-inch, 21-inch, and 36-inch 100-year flood projections", - "category": "flood", - "type": "vector", - "files": [ + }, { "url": "https://data.kitware.com/api/v1/item/64f87bc16725af35134c1488/download", "path": "boston/9in_100yr_flood.geojson" @@ -246,12 +252,87 @@ "path": "boston/36in_100yr_flood.geojson" } ], - "style_options": { - "outline": "white", - "palette": [ - "blue" - ] - } + "layers": [ + { + "name": "Boston Sea Level Rises", + "frames": [ + { + "name": "9in", + "data": "9in_rise.geojson" + }, + { + "name": "21in", + "data": "21in_rise.geojson" + }, + { + "name": "36in", + "data": "36in_rise.geojson" + } + ] + }, + { + "name": "Boston 10-Year Flood Events", + "frames": [ + { + "name": "9in", + "data": "9in_10yr_flood.geojson" + }, + { + "name": "21in", + "data": "21in_10yr_flood.geojson" + }, + { + "name": "36in", + "data": "36in_10yr_flood.geojson" + } + ] + }, + { + "name": "Boston 100-Year Flood Events", + "frames": [ + { + "name": "9in", + "data": "9in_100yr_flood.geojson" + }, + { + "name": "21in", + "data": "21in_100yr_flood.geojson" + }, + { + "name": "36in", + "data": "36in_100yr_flood.geojson" + } + ] + } + ] + }, + { + "name": "Boston Flood Timeseries", + "description": "46-hour flood simulation over the Boston Harbor Watershed", + "category": "flood", + "metadata": { + "source": "Simulated by Jack Watson at Northeastern University" + }, + "files": [ + { + "url": "https://data.kitware.com/api/v1/item/6564cc5ac5a2b36857ad16cf/download", + "path": "boston/flood_simulation.zip" + }, + { + "url": "https://data.kitware.com/api/v1/item/6584bcfe8c54f378b99230b0/download", + "path": "boston/flood_simulation.nc" + } + ], + "layers": [ + { + "name": "45-hour flood from many files", + "source_file": "flood_simulation.zip" + }, + { + "name": "45-hour flood from one multiband file", + "data": "flood_simulation.nc" + } + ] }, { "name": "DC Metro", @@ -260,18 +341,16 @@ "files": [ { "url": "https://data.kitware.com/api/v1/item/64b80188cf6a0eaef6c1416e/download", - "path": "washington/DC_Metro.zip" + "path": "washington/DC_Metro.zip", + "metadata": { + "combine_contents": "true" + } } ], "network_options": { "connection_column": "LINE", "connection_column_delimiter": ", ", "node_id_column": "NAME" - }, - "style_options": { - "color_property": "LINE", - "color_delimiter": ", ", - "outline": "white" } } ] diff --git a/sample_data/use_cases/boston_floods/ingest.py b/sample_data/use_cases/boston_floods/ingest.py index 6d89aec0..a659ee6f 100644 --- a/sample_data/use_cases/boston_floods/ingest.py +++ b/sample_data/use_cases/boston_floods/ingest.py @@ -2,7 +2,7 @@ def convert_dataset(dataset, options): print('\t', f'Converting data for {dataset.name}...') dataset.spawn_conversion_task( - style_options=options.get('style_options'), + layer_options=options.get('layers'), network_options=options.get('network_options'), region_options=options.get('region_options'), asynchronous=False, diff --git a/sample_data/use_cases/boston_floods/projects.json b/sample_data/use_cases/boston_floods/projects.json index 87a8c874..e2258e25 100644 --- a/sample_data/use_cases/boston_floods/projects.json +++ b/sample_data/use_cases/boston_floods/projects.json @@ -16,9 +16,8 @@ "Boston Neighborhoods", "Boston Census 2020 Block Groups", "Boston Zip Codes", - "Boston Sea Level Rises", - "Boston 10-Year Flood Events", - "Boston 100-Year Flood Events" + "Boston Projected Flood Events", + "Boston Flood Timeseries" ] }, { diff --git a/sample_data/use_cases/new_york_energy/datasets.json b/sample_data/use_cases/new_york_energy/datasets.json index 193774c4..590e7bc7 100644 --- a/sample_data/use_cases/new_york_energy/datasets.json +++ b/sample_data/use_cases/new_york_energy/datasets.json @@ -8,7 +8,6 @@ "name": "County Boundaries", "description": "From https://gis.ny.gov/civil-boundaries", "category": "region", - "type": "vector", "files": [ { "url": "https://data.kitware.com/api/v1/item/66a2a19a5d2551c516b1e502/download", @@ -17,12 +16,6 @@ ], "region_options": { "name_property": "NAME" - }, - "style_options": { - "outline": "white", - "palette": [ - "grey" - ] } }, { diff --git a/sample_data/use_cases/new_york_energy/export_networks.py b/sample_data/use_cases/new_york_energy/export_networks.py index 83918540..5aca70a6 100644 --- a/sample_data/use_cases/new_york_energy/export_networks.py +++ b/sample_data/use_cases/new_york_energy/export_networks.py @@ -4,7 +4,7 @@ from datetime import datetime from pathlib import Path -from uvdat.core.models import Network, SourceRegion +from uvdat.core.models import Network, Region OUTPUT_FOLDER = Path('sample_data/use_cases/new_york_energy/networks') @@ -12,8 +12,8 @@ def perform_export(): start = datetime.now() - networks = Network.objects.filter(dataset__name='National Grid Network') - zones = SourceRegion.objects.filter(dataset__name='County Boundaries') + networks = Network.objects.filter(vector_data__dataset__name='National Grid Network') + zones = Region.objects.filter(dataset__name='County Boundaries') for network in networks: sample_node = network.nodes.first() diff --git a/sample_data/use_cases/new_york_energy/import_networks.py b/sample_data/use_cases/new_york_energy/import_networks.py index 1da40b58..5e525bd2 100644 --- a/sample_data/use_cases/new_york_energy/import_networks.py +++ b/sample_data/use_cases/new_york_energy/import_networks.py @@ -8,7 +8,7 @@ from django.contrib.gis.measure import D from django.contrib.gis.db.models.functions import Distance from django.contrib.gis.geos import Point, LineString -from uvdat.core.models import Network, NetworkEdge, NetworkNode, VectorMapLayer +from uvdat.core.models import Network, NetworkEdge, NetworkNode, VectorData from uvdat.core.tasks.networks import create_vector_features_from_network @@ -24,9 +24,14 @@ def get_metadata(feature): def create_network(dataset, network_name, geodata): print(f'\t\tCreating network for {network_name}.') - network = Network.objects.create( + vector_data = VectorData.objects.create( + name=network_name, dataset=dataset, + ) + network = Network.objects.create( + name=network_name, category='energy', + vector_data=vector_data, metadata=dict(name=network_name) ) features = geodata.get('features') @@ -81,8 +86,8 @@ def create_network(dataset, network_name, geodata): def perform_import(dataset, **kwargs): print('\tEstimated time: 45 minutes.') start = datetime.now() - Network.objects.filter(dataset=dataset).delete() - VectorMapLayer.objects.filter(dataset=dataset).delete() + Network.objects.filter(vector_data__dataset=dataset).delete() + VectorData.objects.filter(dataset=dataset).delete() for file_item in dataset.source_files.all(): with tempfile.TemporaryDirectory() as temp_dir: archive_path = Path(temp_dir, 'archive.zip') diff --git a/sample_data/use_cases/new_york_energy/ingest.py b/sample_data/use_cases/new_york_energy/ingest.py index eb959f7c..dc68af45 100644 --- a/sample_data/use_cases/new_york_energy/ingest.py +++ b/sample_data/use_cases/new_york_energy/ingest.py @@ -3,6 +3,8 @@ from .export_networks import perform_export from .nysdp import create_consolidated_network, create_vector_features +from uvdat.core.tasks.dataset import create_layers_and_frames + DOWNLOADS_FOLDER = DOWNLOADS_FOLDER = Path('../../sample_data/downloads') PULL_LATEST = False @@ -18,13 +20,16 @@ def convert_dataset(dataset, options): perform_export() else: perform_import(dataset, downloads_folder=DOWNLOADS_FOLDER) + create_layers_and_frames(dataset) elif dataset.name == 'National Grid CompanyBoundary': create_vector_features(dataset, 'CompanyBoundary') + create_layers_and_frames(dataset) elif dataset.name == 'National Grid Substations': create_vector_features(dataset, 'Substations') + create_layers_and_frames(dataset) else: dataset.spawn_conversion_task( - style_options=options.get('style_options'), + layer_options=options.get('layers'), network_options=options.get('network_options'), region_options=options.get('region_options'), asynchronous=False, diff --git a/sample_data/use_cases/new_york_energy/nysdp.py b/sample_data/use_cases/new_york_energy/nysdp.py index de763420..03e4c97f 100644 --- a/sample_data/use_cases/new_york_energy/nysdp.py +++ b/sample_data/use_cases/new_york_energy/nysdp.py @@ -6,8 +6,18 @@ from pathlib import Path from concurrent.futures import ThreadPoolExecutor -from django.contrib.gis.geos import GEOSGeometry, Point, LineString -from uvdat.core.models import Dataset, Network, NetworkNode, NetworkEdge, VectorMapLayer, VectorFeature, SourceRegion +from django.contrib.gis.geos import GEOSGeometry, LineString, Point +from uvdat.core.models import ( + Dataset, + Layer, + LayerFrame, + Network, + NetworkEdge, + NetworkNode, + Region, + VectorData, + VectorFeature, +) from uvdat.core.tasks.networks import create_vector_features_from_network from .interpret_network import interpret_group @@ -49,16 +59,15 @@ def fetch_vector_features(service_name=None, **kwargs): def create_vector_features(dataset, service_name=None, **kwargs): - VectorMapLayer.objects.filter(dataset=dataset).delete() - + VectorData.objects.filter(dataset=dataset).delete() + vector_data = VectorData.objects.create(dataset=dataset, name=dataset.name) feature_sets = fetch_vector_features(service_name=service_name) vector_features = [] for index, feature_set in feature_sets.items(): - map_layer = VectorMapLayer.objects.create(dataset=dataset, index=index) for feature in feature_set: vector_features.append( VectorFeature( - map_layer=map_layer, + vector_data=vector_data, geometry=GEOSGeometry(json.dumps(feature['geometry'])), properties=feature['properties'], ) @@ -122,14 +131,14 @@ def download_all_deduped_vector_features(**kwargs): def create_consolidated_network(dataset, **kwargs): start = datetime.now() - Network.objects.filter(dataset=dataset).delete() - VectorMapLayer.objects.filter(dataset=network.dataset).delete() + Network.objects.filter(vector_data__dataset=dataset).delete() + VectorData.objects.filter(dataset=network.dataset).delete() gdf = download_all_deduped_vector_features(**kwargs) zones_dataset_name = kwargs.get('zones_dataset_name') if zones_dataset_name is None: raise ValueError('`zones_dataset_name` is required.') - zones = SourceRegion.objects.filter(dataset__name=zones_dataset_name) + zones = Region.objects.filter(dataset__name=zones_dataset_name) if zones.count() == 0: raise ValueError(f'No regions found with dataset name "{zones_dataset_name}".') @@ -148,9 +157,17 @@ def create_consolidated_network(dataset, **kwargs): with ThreadPoolExecutor(max_workers=10) as pool: results = pool.map(interpret_group, groups) - for result in results: + for i, result in enumerate(results): nodes, edges = result - network = Network.objects.create(dataset=dataset) + vector_data = VectorData.objects.create( + name=f'{dataset.name} Network {i}', + dataset=dataset, + ) + network = Network.objects.create( + name=vector_data.name, + vector_data=vector_data, + category='energy' + ) NetworkNode.objects.bulk_create([ NetworkNode( network=network, diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index c2999d60..7815226c 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -4,15 +4,17 @@ Chart, Dataset, FileItem, + Layer, + LayerFrame, Network, NetworkEdge, NetworkNode, Project, - RasterMapLayer, + RasterData, + Region, SimulationResult, - SourceRegion, + VectorData, VectorFeature, - VectorMapLayer, ) @@ -21,7 +23,7 @@ class ProjectAdmin(admin.ModelAdmin): class DatasetAdmin(admin.ModelAdmin): - list_display = ['id', 'name', 'dataset_type', 'category'] + list_display = ['id', 'name', 'category'] class FileItemAdmin(admin.ModelAdmin): @@ -39,31 +41,46 @@ class ChartAdmin(admin.ModelAdmin): list_display = ['id', 'name', 'editable'] -class RasterMapLayerAdmin(admin.ModelAdmin): - list_display = ['id', 'get_dataset_name', 'index'] +class LayerAdmin(admin.ModelAdmin): + list_display = ['id', 'name', 'get_dataset_name'] def get_dataset_name(self, obj): return obj.dataset.name -class VectorMapLayerAdmin(admin.ModelAdmin): - list_display = ['id', 'get_dataset_name', 'index'] +class LayerFrameAdmin(admin.ModelAdmin): + list_display = ['id', 'name', 'index', 'get_layer_name'] - def get_dataset_name(self, obj): - return obj.dataset.name + def get_layer_name(self, obj): + return obj.layer.name + + +class RasterDataAdmin(admin.ModelAdmin): + list_display = ['id', 'name', 'get_source_file_name'] + + def get_source_file_name(self, obj): + if obj.source_file is not None: + return obj.source_file.name + return '' + + +class VectorDataAdmin(admin.ModelAdmin): + list_display = ['id', 'name', 'get_source_file_name'] + + def get_source_file_name(self, obj): + if obj.source_file is not None: + return obj.source_file.name + return '' class VectorFeatureAdmin(admin.ModelAdmin): - list_display = ['id', 'get_dataset_name', 'get_map_layer_index'] + list_display = ['id', 'get_dataset_name'] def get_dataset_name(self, obj): - return obj.map_layer.dataset.name - - def get_map_layer_index(self, obj): - return obj.map_layer.index + return obj.vector_data.dataset.name -class SourceRegionAdmin(admin.ModelAdmin): +class RegionAdmin(admin.ModelAdmin): list_display = ['id', 'name', 'get_dataset_name'] def get_dataset_name(self, obj): @@ -84,7 +101,7 @@ def get_network_id(self, obj): return obj.network.id def get_dataset_name(self, obj): - return obj.network.dataset.name + return obj.network.vector_data.dataset.name class NetworkNodeAdmin(admin.ModelAdmin): @@ -94,7 +111,7 @@ def get_network_id(self, obj): return obj.network.id def get_dataset_name(self, obj): - return obj.network.dataset.name + return obj.network.vector_data.dataset.name def get_adjacent_node_names(self, obj): return ', '.join(n.name for n in obj.get_adjacent_nodes()) @@ -108,10 +125,12 @@ class SimulationResultAdmin(admin.ModelAdmin): admin.site.register(Dataset, DatasetAdmin) admin.site.register(FileItem, FileItemAdmin) admin.site.register(Chart, ChartAdmin) -admin.site.register(RasterMapLayer, RasterMapLayerAdmin) -admin.site.register(VectorMapLayer, VectorMapLayerAdmin) +admin.site.register(Layer, LayerAdmin) +admin.site.register(LayerFrame, LayerFrameAdmin) +admin.site.register(RasterData, RasterDataAdmin) +admin.site.register(VectorData, VectorDataAdmin) admin.site.register(VectorFeature, VectorFeatureAdmin) -admin.site.register(SourceRegion, SourceRegionAdmin) +admin.site.register(Region, RegionAdmin) admin.site.register(Network, NetworkAdmin) admin.site.register(NetworkNode, NetworkNodeAdmin) admin.site.register(NetworkEdge, NetworkEdgeAdmin) diff --git a/uvdat/core/migrations/0010_layers_and_frames.py b/uvdat/core/migrations/0010_layers_and_frames.py new file mode 100644 index 00000000..b7a70092 --- /dev/null +++ b/uvdat/core/migrations/0010_layers_and_frames.py @@ -0,0 +1,256 @@ +# Generated by Django 5.0.7 on 2025-01-31 18:49 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import django.db.models.deletion +import s3_file_field.fields + +import uvdat.core.models.layer + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0009_file_size_field'), + ] + + operations = [ + migrations.RemoveField( + model_name='sourceregion', + name='dataset', + ), + migrations.RemoveField( + model_name='vectormaplayer', + name='dataset', + ), + migrations.RemoveField( + model_name='vectorfeature', + name='map_layer', + ), + migrations.RemoveField( + model_name='dataset', + name='dataset_type', + ), + migrations.RemoveField( + model_name='network', + name='dataset', + ), + migrations.AddField( + model_name='network', + name='name', + field=models.CharField(default='Network', max_length=255), + ), + migrations.AddField( + model_name='networkedge', + name='vector_feature', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='edges', + to='core.vectorfeature', + ), + ), + migrations.AddField( + model_name='networknode', + name='vector_feature', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='nodes', + to='core.vectorfeature', + ), + ), + migrations.CreateModel( + name='Layer', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('name', models.CharField(default='Layer', max_length=255)), + ('metadata', models.JSONField(blank=True, null=True)), + ( + 'dataset', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='layers', + to='core.dataset', + ), + ), + ], + ), + migrations.CreateModel( + name='RasterData', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('name', models.CharField(default='Raster Data', max_length=255)), + ('cloud_optimized_geotiff', s3_file_field.fields.S3FileField(null=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ( + 'dataset', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='rasters', + to='core.dataset', + ), + ), + ( + 'source_file', + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.fileitem' + ), + ), + ], + ), + migrations.CreateModel( + name='Region', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('name', models.CharField(max_length=255)), + ('metadata', models.JSONField(blank=True, null=True)), + ('boundary', django.contrib.gis.db.models.fields.MultiPolygonField(srid=4326)), + ( + 'dataset', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='regions', + to='core.dataset', + ), + ), + ( + 'vector_feature', + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='regions', + to='core.vectorfeature', + ), + ), + ], + ), + migrations.CreateModel( + name='VectorData', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('name', models.CharField(default='Vector Data', max_length=255)), + ('geojson_data', s3_file_field.fields.S3FileField(null=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ( + 'dataset', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='vectors', + to='core.dataset', + ), + ), + ( + 'source_file', + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.fileitem' + ), + ), + ], + ), + migrations.CreateModel( + name='LayerFrame', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('name', models.CharField(default='Layer Frame', max_length=255)), + ('index', models.PositiveIntegerField(default=0)), + ( + 'source_filters', + models.JSONField(default=uvdat.core.models.layer.default_source_filters), + ), + ('metadata', models.JSONField(blank=True, null=True)), + ( + 'layer', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='frames', + to='core.layer', + ), + ), + ( + 'raster', + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.rasterdata' + ), + ), + ( + 'vector', + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.vectordata' + ), + ), + ], + ), + migrations.AddField( + model_name='network', + name='vector_data', + field=models.ForeignKey( + default=None, + on_delete=django.db.models.deletion.CASCADE, + related_name='networks', + to='core.vectordata', + ), + preserve_default=False, + ), + migrations.AddField( + model_name='vectorfeature', + name='vector_data', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='features', + to='core.vectordata', + ), + ), + migrations.DeleteModel( + name='RasterMapLayer', + ), + migrations.DeleteModel( + name='SourceRegion', + ), + migrations.DeleteModel( + name='VectorMapLayer', + ), + migrations.AddConstraint( + model_name='region', + constraint=models.UniqueConstraint( + fields=('dataset', 'name'), name='unique-source-region-name' + ), + ), + migrations.AddConstraint( + model_name='layerframe', + constraint=models.CheckConstraint( + check=models.Q( + models.Q(('raster__isnull', False), ('vector__isnull', True)), + models.Q(('raster__isnull', True), ('vector__isnull', False)), + _connector='OR', + ), + name='exactly_one_data', + ), + ), + ] diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index 11b3ac1d..458d6cda 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -1,10 +1,11 @@ from .chart import Chart +from .data import RasterData, VectorData, VectorFeature from .dataset import Dataset from .file_item import FileItem -from .map_layers import RasterMapLayer, VectorFeature, VectorMapLayer +from .layer import Layer, LayerFrame from .networks import Network, NetworkEdge, NetworkNode from .project import Project -from .regions import SourceRegion +from .regions import Region from .simulations import SimulationResult __all__ = [ @@ -12,10 +13,12 @@ Project, Dataset, FileItem, - RasterMapLayer, - VectorMapLayer, + RasterData, + VectorData, VectorFeature, - SourceRegion, + Layer, + LayerFrame, + Region, Network, NetworkEdge, NetworkNode, diff --git a/uvdat/core/models/map_layers.py b/uvdat/core/models/data.py similarity index 57% rename from uvdat/core/models/map_layers.py rename to uvdat/core/models/data.py index c5642149..23bbe43b 100644 --- a/uvdat/core/models/map_layers.py +++ b/uvdat/core/models/data.py @@ -6,26 +6,19 @@ from django.core.files.base import ContentFile from django.db import models from django.dispatch import receiver -from django_extensions.db.models import TimeStampedModel import large_image from s3_file_field import S3FileField from .dataset import Dataset +from .file_item import FileItem -class AbstractMapLayer(TimeStampedModel): - dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, null=True) +class RasterData(models.Model): + name = models.CharField(max_length=255, default='Raster Data') + dataset = models.ForeignKey(Dataset, related_name='rasters', on_delete=models.CASCADE) + source_file = models.ForeignKey(FileItem, null=True, on_delete=models.CASCADE) + cloud_optimized_geotiff = S3FileField(null=True) metadata = models.JSONField(blank=True, null=True) - default_style = models.JSONField(blank=True, null=True) - index = models.IntegerField(null=True) - - class Meta: - abstract = True - - -class RasterMapLayer(AbstractMapLayer): - cloud_optimized_geotiff = S3FileField() - # TODO: Store data x/y min/max bounds on model def get_image_data(self, resolution: float = 1.0): with tempfile.TemporaryDirectory() as tmp: @@ -41,8 +34,12 @@ def get_image_data(self, resolution: float = 1.0): return data.tolist() -class VectorMapLayer(AbstractMapLayer): - geojson_file = S3FileField(null=True) +class VectorData(models.Model): + name = models.CharField(max_length=255, default='Vector Data') + dataset = models.ForeignKey(Dataset, related_name='vectors', on_delete=models.CASCADE) + source_file = models.ForeignKey(FileItem, null=True, on_delete=models.CASCADE) + geojson_data = S3FileField(null=True) + metadata = models.JSONField(blank=True, null=True) def write_geojson_data(self, content: str | dict): if isinstance(content, str): @@ -52,26 +49,28 @@ def write_geojson_data(self, content: str | dict): else: raise Exception(f'Invalid content type supplied: {type(content)}') - self.geojson_file.save('vectordata.geojson', ContentFile(data.encode())) + self.geojson_data.save('vectordata.geojson', ContentFile(data.encode())) def read_geojson_data(self) -> dict: - """Read and load the data from geojson_file into a dict.""" - return json.load(self.geojson_file.open()) + """Read and load the data from geojson_data into a dict.""" + return json.load(self.geojson_data.open()) class VectorFeature(models.Model): - map_layer = models.ForeignKey(VectorMapLayer, on_delete=models.CASCADE) + vector_data = models.ForeignKey( + VectorData, on_delete=models.CASCADE, related_name='features', null=True + ) geometry = geomodels.GeometryField() properties = models.JSONField() -@receiver(models.signals.post_delete, sender=RasterMapLayer) +@receiver(models.signals.post_delete, sender=RasterData) def delete_raster_content(sender, instance, **kwargs): if instance.cloud_optimized_geotiff: instance.cloud_optimized_geotiff.delete(save=False) -@receiver(models.signals.post_delete, sender=VectorMapLayer) +@receiver(models.signals.post_delete, sender=VectorData) def delete_vector_content(sender, instance, **kwargs): - if instance.geojson_file: - instance.geojson_file.delete(save=False) + if instance.geojson_data: + instance.geojson_data.delete(save=False) diff --git a/uvdat/core/models/dataset.py b/uvdat/core/models/dataset.py index 9e656b64..b5bf4399 100644 --- a/uvdat/core/models/dataset.py +++ b/uvdat/core/models/dataset.py @@ -2,23 +2,15 @@ class Dataset(models.Model): - class DatasetType(models.TextChoices): - VECTOR = 'VECTOR', 'Vector' - RASTER = 'RASTER', 'Raster' - name = models.CharField(max_length=255, unique=True) description = models.TextField(null=True, blank=True) category = models.CharField(max_length=25) processing = models.BooleanField(default=False) metadata = models.JSONField(blank=True, null=True) - dataset_type = models.CharField( - max_length=max(len(choice[0]) for choice in DatasetType.choices), - choices=DatasetType.choices, - ) def spawn_conversion_task( self, - style_options=None, + layer_options=None, network_options=None, region_options=None, asynchronous=True, @@ -26,9 +18,9 @@ def spawn_conversion_task( from uvdat.core.tasks.dataset import convert_dataset if asynchronous: - convert_dataset.delay(self.id, style_options, network_options, region_options) + convert_dataset.delay(self.id, layer_options, network_options, region_options) else: - convert_dataset(self.id, style_options, network_options, region_options) + convert_dataset(self.id, layer_options, network_options, region_options) def get_size(self): from uvdat.core.models import FileItem @@ -39,18 +31,12 @@ def get_size(self): size += file_item.file_size return size - def get_regions(self): - from uvdat.core.models import SourceRegion - - return SourceRegion.objects.filter(dataset=self) + def get_networks(self): + from uvdat.core.models import Network - def get_map_layers(self): - """Return a queryset of either RasterMapLayer, or VectorMapLayer.""" - from uvdat.core.models import RasterMapLayer, VectorMapLayer + return Network.objects.filter(vector_data__dataset=self) - if self.dataset_type == self.DatasetType.RASTER: - return RasterMapLayer.objects.filter(dataset=self) - if self.dataset_type == self.DatasetType.VECTOR: - return VectorMapLayer.objects.filter(dataset=self) + def get_regions(self): + from uvdat.core.models import Region - raise NotImplementedError(f'Dataset Type {self.dataset_type}') + return Region.objects.filter(dataset=self) diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py new file mode 100644 index 00000000..92351116 --- /dev/null +++ b/uvdat/core/models/layer.py @@ -0,0 +1,38 @@ +from django.db import models + +from .data import RasterData, VectorData +from .dataset import Dataset + + +def default_source_filters(): + return dict(band=1) + + +class Layer(models.Model): + name = models.CharField(max_length=255, default='Layer') + dataset = models.ForeignKey(Dataset, related_name='layers', on_delete=models.CASCADE) + metadata = models.JSONField(blank=True, null=True) + + +class LayerFrame(models.Model): + name = models.CharField(max_length=255, default='Layer Frame') + layer = models.ForeignKey(Layer, related_name='frames', on_delete=models.CASCADE) + vector = models.ForeignKey(VectorData, null=True, on_delete=models.CASCADE) + raster = models.ForeignKey(RasterData, null=True, on_delete=models.CASCADE) + index = models.PositiveIntegerField(default=0) + source_filters = models.JSONField(default=default_source_filters) + metadata = models.JSONField(blank=True, null=True) + + class Meta: + constraints = [ + models.CheckConstraint( + check=(models.Q(raster__isnull=False) & models.Q(vector__isnull=True)) + | (models.Q(raster__isnull=True) & models.Q(vector__isnull=False)), + name='exactly_one_data', + ) + ] + + def get_data(self): + if self.raster is not None: + return self.raster + return self.vector diff --git a/uvdat/core/models/networks.py b/uvdat/core/models/networks.py index 5bafd566..d35e676f 100644 --- a/uvdat/core/models/networks.py +++ b/uvdat/core/models/networks.py @@ -1,7 +1,7 @@ from django.contrib.gis.db import models as geo_models from django.db import connection, models -from .dataset import Dataset +from .data import VectorData, VectorFeature GCC_QUERY = """ WITH RECURSIVE n as ( @@ -43,7 +43,8 @@ class Network(models.Model): - dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='networks') + name = models.CharField(max_length=255, default='Network') + vector_data = models.ForeignKey(VectorData, on_delete=models.CASCADE, related_name='networks') category = models.CharField(max_length=25) metadata = models.JSONField(blank=True, null=True) @@ -81,6 +82,9 @@ def get_gcc(self, excluded_nodes: list[int]): class NetworkNode(models.Model): name = models.CharField(max_length=255) + vector_feature = models.ForeignKey( + VectorFeature, on_delete=models.CASCADE, related_name='nodes', null=True + ) network = models.ForeignKey(Network, on_delete=models.CASCADE, related_name='nodes') metadata = models.JSONField(blank=True, null=True) capacity = models.IntegerField(null=True) @@ -104,6 +108,9 @@ def get_adjacent_nodes(self) -> models.QuerySet: class NetworkEdge(models.Model): name = models.CharField(max_length=255) + vector_feature = models.ForeignKey( + VectorFeature, on_delete=models.CASCADE, related_name='edges', null=True + ) network = models.ForeignKey(Network, on_delete=models.CASCADE, related_name='edges') metadata = models.JSONField(blank=True, null=True) capacity = models.IntegerField(null=True) diff --git a/uvdat/core/models/regions.py b/uvdat/core/models/regions.py index 00948171..1a5bfb40 100644 --- a/uvdat/core/models/regions.py +++ b/uvdat/core/models/regions.py @@ -1,11 +1,15 @@ from django.contrib.gis.db import models as geo_models from django.db import models +from .data import VectorFeature from .dataset import Dataset -class SourceRegion(models.Model): +class Region(models.Model): name = models.CharField(max_length=255) + vector_feature = models.ForeignKey( + VectorFeature, on_delete=models.CASCADE, related_name='regions', null=True + ) dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='regions') metadata = models.JSONField(blank=True, null=True) boundary = geo_models.MultiPolygonField() diff --git a/uvdat/core/models/simulations.py b/uvdat/core/models/simulations.py index 6115a23a..27ccbdb3 100644 --- a/uvdat/core/models/simulations.py +++ b/uvdat/core/models/simulations.py @@ -3,7 +3,7 @@ from uvdat.core.tasks import simulations as uvdat_simulations -from .map_layers import RasterMapLayer, VectorMapLayer +from .data import RasterData, VectorData from .networks import Network from .project import Project @@ -66,12 +66,12 @@ def run(self, **kwargs): }, { 'name': 'elevation_data', - 'type': RasterMapLayer, + 'type': RasterData, 'options_query': {'dataset__category': 'elevation'}, }, { 'name': 'flood_area', - 'type': VectorMapLayer, + 'type': VectorData, 'options_query': {'dataset__category': 'flood'}, }, ], @@ -112,7 +112,7 @@ def run(self, **kwargs): 'args': [ { 'name': 'imagery_layer', - 'type': RasterMapLayer, + 'type': RasterData, 'options_query': {'dataset__category': 'imagery'}, } ], diff --git a/uvdat/core/rest/__init__.py b/uvdat/core/rest/__init__.py index 4fbf6b9c..a3edfdb2 100644 --- a/uvdat/core/rest/__init__.py +++ b/uvdat/core/rest/__init__.py @@ -1,18 +1,21 @@ from .chart import ChartViewSet +from .data import RasterDataViewSet, VectorDataViewSet from .dataset import DatasetViewSet -from .map_layers import RasterMapLayerViewSet, VectorMapLayerViewSet +from .layer import LayerFrameViewSet, LayerViewSet from .project import ProjectViewSet -from .regions import SourceRegionViewSet +from .regions import RegionViewSet from .simulations import SimulationViewSet from .user import UserViewSet __all__ = [ ProjectViewSet, ChartViewSet, - RasterMapLayerViewSet, - VectorMapLayerViewSet, + LayerViewSet, + LayerFrameViewSet, + RasterDataViewSet, + VectorDataViewSet, DatasetViewSet, - SourceRegionViewSet, + RegionViewSet, SimulationViewSet, UserViewSet, ] diff --git a/uvdat/core/rest/access_control.py b/uvdat/core/rest/access_control.py index 980b2b41..6ab5273e 100644 --- a/uvdat/core/rest/access_control.py +++ b/uvdat/core/rest/access_control.py @@ -14,6 +14,9 @@ def filter_queryset_by_projects(queryset: QuerySet[Model], projects: QuerySet[mo # Dataset permissions not yet implemented, and as such, all datasets are visible to all users if model == models.Dataset: return queryset + # RasterData and VectorData permissions should inherit from Dataset permissions + if model in [models.RasterData, models.VectorData]: + return queryset if model == models.Project: return queryset.filter(id__in=projects.values_list('id', flat=True)) @@ -21,12 +24,14 @@ def filter_queryset_by_projects(queryset: QuerySet[Model], projects: QuerySet[mo return queryset.filter(project__in=projects) if model in [ models.FileItem, - models.RasterMapLayer, - models.VectorMapLayer, - models.Network, - models.SourceRegion, + models.Layer, + models.Region, ]: return queryset.filter(dataset__project__in=projects) + if model == models.LayerFrame: + return queryset.filter(layer__dataset__project__in=projects) + if model == models.Network: + return queryset.filter(vector_data__dataset__project__in=projects) if model in [models.NetworkNode, models.NetworkEdge]: return queryset.filter(network__dataset__project__in=projects) diff --git a/uvdat/core/rest/map_layers.py b/uvdat/core/rest/data.py similarity index 66% rename from uvdat/core/rest/map_layers.py rename to uvdat/core/rest/data.py index 2a360883..63290253 100644 --- a/uvdat/core/rest/map_layers.py +++ b/uvdat/core/rest/data.py @@ -8,13 +8,10 @@ from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet -from uvdat.core.models import RasterMapLayer, VectorMapLayer +from uvdat.core.models import RasterData, VectorData from uvdat.core.rest.access_control import GuardianFilter, GuardianPermission -from uvdat.core.rest.serializers import ( - RasterMapLayerSerializer, - VectorMapLayerDetailSerializer, - VectorMapLayerSerializer, -) +from uvdat.core.rest.serializers import RasterDataSerializer, VectorDataSerializer +from uvdat.core.rest.tokenauth import TokenAuth VECTOR_TILE_SQL = """ WITH @@ -60,21 +57,34 @@ core_vectorfeature t, bounds WHERE - t.map_layer_id = %(map_layer_id)s + t.vector_data_id = %(vector_data_id)s AND ST_Intersects( ST_Transform(t.geometry, %(srid)s), ST_Transform(bounds.geom, %(srid)s) ) + REPLACE_WITH_FILTERS ) SELECT ST_AsMVT(mvtgeom.*) FROM mvtgeom ; """ -class RasterMapLayerViewSet(GenericViewSet, mixins.RetrieveModelMixin, LargeImageFileDetailMixin): - queryset = RasterMapLayer.objects.select_related('dataset').all() - serializer_class = RasterMapLayerSerializer +def get_filter_string(filters: dict | None = None): + if filters is None: + return '' + + return_str = '' + for key, value in filters.items(): + key_path = key.replace('.', ',') + return_str += f" AND t.properties #>> '{{{key_path}}}' = '{value}'" + return return_str + + +class RasterDataViewSet(GenericViewSet, mixins.RetrieveModelMixin, LargeImageFileDetailMixin): + queryset = RasterData.objects.select_related('dataset').all() + serializer_class = RasterDataSerializer permission_classes = [GuardianPermission] + authentication_classes = [TokenAuth] filter_backends = [GuardianFilter] lookup_field = 'id' FILE_FIELD_NAME = 'cloud_optimized_geotiff' @@ -86,21 +96,22 @@ class RasterMapLayerViewSet(GenericViewSet, mixins.RetrieveModelMixin, LargeImag url_name='raster_data', ) def get_raster_data(self, request, resolution: str = '1', **kwargs): - raster_map_layer = self.get_object() - data = raster_map_layer.get_image_data(float(resolution)) + raster_data = self.get_object() + data = raster_data.get_image_data(float(resolution)) return HttpResponse(json.dumps(data), status=200) -class VectorMapLayerViewSet(GenericViewSet, mixins.RetrieveModelMixin): - queryset = VectorMapLayer.objects.select_related('dataset').all() - serializer_class = VectorMapLayerSerializer +class VectorDataViewSet(GenericViewSet, mixins.RetrieveModelMixin): + queryset = VectorData.objects.select_related('dataset').all() + serializer_class = VectorDataSerializer permission_classes = [GuardianPermission] + authentication_classes = [TokenAuth] filter_backends = [GuardianFilter] lookup_field = 'id' def retrieve(self, request, *args, **kwargs): instance = self.get_object() - serializer = VectorMapLayerDetailSerializer(instance) + serializer = VectorDataSerializer(instance) return Response(serializer.data) @action( @@ -110,15 +121,18 @@ def retrieve(self, request, *args, **kwargs): url_name='tiles', ) def get_vector_tile(self, request, id: str, x: str, y: str, z: str): + filters = request.query_params.copy() + filters.pop('token', None) + filters_string = get_filter_string(filters) with connection.cursor() as cursor: cursor.execute( - VECTOR_TILE_SQL, + VECTOR_TILE_SQL.replace('REPLACE_WITH_FILTERS', filters_string), { 'z': z, 'x': x, 'y': y, 'srid': 3857, - 'map_layer_id': id, + 'vector_data_id': id, }, ) row = cursor.fetchone() diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index dabcdf29..d40b8f5a 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -8,10 +8,11 @@ from uvdat.core.rest.access_control import GuardianFilter, GuardianPermission from uvdat.core.rest.serializers import ( DatasetSerializer, + LayerSerializer, NetworkEdgeSerializer, NetworkNodeSerializer, - RasterMapLayerSerializer, - VectorMapLayerSerializer, + RasterDataSerializer, + VectorDataSerializer, ) from uvdat.core.tasks.chart import add_gcc_chart_datum @@ -37,27 +38,28 @@ def get_queryset(self): return qs.filter(project=int(project_id)) @action(detail=True, methods=['get']) - def map_layers(self, request, **kwargs): + def layers(self, request, **kwargs): dataset: Dataset = self.get_object() - map_layers = list(dataset.get_map_layers().select_related('dataset')) - - # Set serializer based on dataset type - if dataset.dataset_type == Dataset.DatasetType.RASTER: - serializer = RasterMapLayerSerializer(map_layers, many=True) - elif dataset.dataset_type == Dataset.DatasetType.VECTOR: - # Set serializer - serializer = VectorMapLayerSerializer(map_layers, many=True) - else: - raise NotImplementedError(f'Dataset Type {dataset.dataset_type}') - - # Return response with rendered data + layers = list(dataset.layers.all()) + serializer = LayerSerializer(layers, many=True) return Response(serializer.data, status=200) + @action(detail=True, methods=['get']) + def data(self, request, **kwargs): + dataset: Dataset = self.get_object() + + data = [] + for raster in dataset.rasters.all(): + data.append(RasterDataSerializer(raster).data) + for vector in dataset.vectors.all(): + data.append(VectorDataSerializer(vector).data) + return Response(data, status=200) + @action(detail=True, methods=['get']) def network(self, request, **kwargs): dataset = self.get_object() networks = [] - for network in dataset.networks.all(): + for network in dataset.get_networks().all(): networks.append( { 'nodes': [ @@ -83,12 +85,12 @@ def gcc(self, request, **kwargs): project_id = serializer.validated_data['project'] exclude_nodes = [int(n) for n in serializer.validated_data['exclude_nodes'].split(',')] - if not dataset.networks.exists(): + if not dataset.get_networks().exists(): return Response(data='No networks exist in selected dataset', status=400) # Find the GCC for each network in the dataset network_gccs: list[list[int]] = [] - for network in dataset.networks.all(): + for network in dataset.get_networks().all(): network: Network network_gccs.append(network.get_gcc(excluded_nodes=exclude_nodes)) diff --git a/uvdat/core/rest/layer.py b/uvdat/core/rest/layer.py new file mode 100644 index 00000000..902c043d --- /dev/null +++ b/uvdat/core/rest/layer.py @@ -0,0 +1,21 @@ +from rest_framework.viewsets import ReadOnlyModelViewSet + +from uvdat.core.models import Layer, LayerFrame +from uvdat.core.rest.access_control import GuardianFilter, GuardianPermission +from uvdat.core.rest.serializers import LayerFrameSerializer, LayerSerializer + + +class LayerViewSet(ReadOnlyModelViewSet): + queryset = Layer.objects.select_related('dataset').all() + serializer_class = LayerSerializer + permission_classes = [GuardianPermission] + filter_backends = [GuardianFilter] + lookup_field = 'id' + + +class LayerFrameViewSet(ReadOnlyModelViewSet): + queryset = LayerFrame.objects.select_related('dataset').all() + serializer_class = LayerFrameSerializer + permission_classes = [GuardianPermission] + filter_backends = [GuardianFilter] + lookup_field = 'id' diff --git a/uvdat/core/rest/regions.py b/uvdat/core/rest/regions.py index cbfd60a5..05f54c09 100644 --- a/uvdat/core/rest/regions.py +++ b/uvdat/core/rest/regions.py @@ -1,15 +1,15 @@ from rest_framework import mixins from rest_framework.viewsets import GenericViewSet -from uvdat.core.models import SourceRegion +from uvdat.core.models import Region from uvdat.core.rest.access_control import GuardianFilter, GuardianPermission -from .serializers import SourceRegionSerializer +from .serializers import RegionSerializer -class SourceRegionViewSet(mixins.RetrieveModelMixin, mixins.ListModelMixin, GenericViewSet): - queryset = SourceRegion.objects.all() - serializer_class = SourceRegionSerializer +class RegionViewSet(mixins.RetrieveModelMixin, mixins.ListModelMixin, GenericViewSet): + queryset = Region.objects.all() + serializer_class = RegionSerializer permission_classes = [GuardianPermission] filter_backends = [GuardianFilter] lookup_field = 'id' diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index de30f7cd..65508674 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -7,14 +7,16 @@ Chart, Dataset, FileItem, + Layer, + LayerFrame, Network, NetworkEdge, NetworkNode, Project, - RasterMapLayer, + RasterData, + Region, SimulationResult, - SourceRegion, - VectorMapLayer, + VectorData, ) @@ -100,70 +102,34 @@ class Meta: fields = '__all__' -class AbstractMapLayerSerializer(serializers.Serializer): - name = serializers.SerializerMethodField('get_name') - type = serializers.SerializerMethodField('get_type') - dataset_id = serializers.SerializerMethodField('get_dataset_id') - file_item = serializers.SerializerMethodField('get_file_item') - - def get_name(self, obj: VectorMapLayer | RasterMapLayer): - if obj.dataset: - for file_item in obj.dataset.source_files.all(): - if file_item.index == obj.index: - return file_item.name - return f'{obj.dataset.name} Layer {obj.index}' - return None - - def get_type(self, obj: VectorMapLayer | RasterMapLayer): - if isinstance(obj, VectorMapLayer): - return 'vector' - return 'raster' - - def get_dataset_id(self, obj: VectorMapLayer | RasterMapLayer): - if obj.dataset: - return obj.dataset.id - return None - - def get_file_item(self, obj: VectorMapLayer | RasterMapLayer): - if obj.dataset is None: - return None - for file_item in obj.dataset.source_files.all(): - if file_item.index == obj.index: - return { - 'id': file_item.id, - 'name': file_item.name, - } - - -class RasterMapLayerSerializer(serializers.ModelSerializer, AbstractMapLayerSerializer): +class LayerSerializer(serializers.ModelSerializer): class Meta: - model = RasterMapLayer - fields = '__all__' - + model = Layer + depth = 2 + fields = ['id', 'name', 'frames', 'metadata', 'dataset'] -class VectorMapLayerSerializer(serializers.ModelSerializer, AbstractMapLayerSerializer): - dataset_category = serializers.SerializerMethodField() - def get_dataset_category(self, obj: VectorMapLayer): - if obj.dataset is None: - raise Exception('map layer with null dataset!') +class LayerFrameSerializer(serializers.ModelSerializer): + class Meta: + model = LayerFrame + fields = '__all__' - return obj.dataset.category +class VectorDataSerializer(serializers.ModelSerializer): class Meta: - model = VectorMapLayer - exclude = ['geojson_file'] + model = VectorData + fields = '__all__' -class VectorMapLayerDetailSerializer(VectorMapLayerSerializer): +class RasterDataSerializer(serializers.ModelSerializer): class Meta: - model = VectorMapLayer - exclude = ['geojson_file'] + model = RasterData + fields = '__all__' -class SourceRegionSerializer(serializers.ModelSerializer): +class RegionSerializer(serializers.ModelSerializer): class Meta: - model = SourceRegion + model = Region fields = '__all__' @@ -181,11 +147,6 @@ class Meta: model = Network fields = '__all__' - name = serializers.SerializerMethodField('get_name') - - def get_name(self, obj): - return obj.dataset.name - class NetworkNodeSerializer(serializers.ModelSerializer): class Meta: diff --git a/uvdat/core/rest/tokenauth.py b/uvdat/core/rest/tokenauth.py new file mode 100644 index 00000000..eef34404 --- /dev/null +++ b/uvdat/core/rest/tokenauth.py @@ -0,0 +1,23 @@ +# Created for the UVDAT Data Explorer, which uses ipyleaflet +# ipyleaflet does not support custom headers, so auth token must be sent as a URL param + +from django.contrib.auth.models import User +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + + +class TokenAuth(OAuth2Authentication): + def authenticate(self, request): + token = request.query_params.get('token') + if token is None: + token_string = request.headers.get('Authorization') + token = token_string.replace('Token ', '') + if token is not None: + try: + user = User.objects.get(auth_token=token) + except User.DoesNotExist: + return None + + if not user.is_active: + return None + return (user, None) + return None diff --git a/uvdat/core/tasks/chart.py b/uvdat/core/tasks/chart.py index 638432a1..d576d442 100644 --- a/uvdat/core/tasks/chart.py +++ b/uvdat/core/tasks/chart.py @@ -63,7 +63,10 @@ def get_gcc_chart(dataset, project_id): 'chart_title': 'Size of Greatest Connected Component over Period', 'x_title': 'Step when Excluded Nodes Changed', 'y_title': 'Number of Nodes', - 'y_range': [0, NetworkNode.objects.filter(network__dataset=dataset).count()], + 'y_range': [ + 0, + NetworkNode.objects.filter(network__vector_data__dataset=dataset).count(), + ], }, ) print('\t', f'Chart {chart.name} created.') diff --git a/uvdat/core/tasks/conversion.py b/uvdat/core/tasks/conversion.py new file mode 100644 index 00000000..174a205c --- /dev/null +++ b/uvdat/core/tasks/conversion.py @@ -0,0 +1,152 @@ +import json +from pathlib import Path +import tempfile +import zipfile + +from django.core.files.base import ContentFile +import geopandas +import large_image +import large_image_converter +import numpy +import rasterio +import shapefile + +from uvdat.core.models import RasterData, VectorData + +RASTER_FILETYPES = ['tif', 'tiff', 'nc', 'jp2'] +IGNORE_FILETYPES = ['dbf', 'sbn', 'sbx', 'cpg', 'shp.xml', 'shx', 'vrt', 'hdf', 'lyr'] + + +def get_cog_path(file): + raster_path = None + try: + # if large_image can open file and geospatial is True, rasterio is not needed. + source = large_image.open(file) + if source.geospatial: + raster_path = file + except large_image.exceptions.TileSourceError: + pass + + if raster_path is None: + # if original data cannot be interpreted by large_image, use rasterio + raster_path = file.parent / 'rasterio.tiff' + with open(file, 'rb') as f: + input_data = rasterio.open(f) + output_data = rasterio.open( + raster_path, + 'w', + driver='GTiff', + height=input_data.height, + width=input_data.width, + count=1, + dtype=numpy.float32, + crs=input_data.crs, + transform=input_data.transform, + ) + band = input_data.read(1) + output_data.write(band, 1) + output_data.close() + + cog_path = file.parent / file.name.replace(file.suffix, 'tiff') + # use large_image to convert new raster data to COG + large_image_converter.convert(str(raster_path), str(cog_path), overwrite=True) + return cog_path + + +def convert_files(*files, file_item=None, combine=False): + source_projection = 4326 + geodata_set = [] + cog_set = [] + metadata = dict(source_filenames=[]) + for file in files: + metadata.update(file_item.metadata) + metadata['source_filenames'].append(file_item.name) + if file.name.endswith('.prj'): + with open(file, 'rb') as f: + contents = f.read() + source_projection = contents.decode() + continue + elif file.name.endswith('.shp'): + reader = shapefile.Reader(file) + geodata_set.append(dict(name=file.name, features=reader.__geo_interface__['features'])) + elif any(file.name.endswith(suffix) for suffix in ['.json', '.geojson']): + with open(file, 'rb') as f: + data = json.load(f) + geodata_set.append(dict(name=file.name, features=data.get('features'))) + source_projection = data.get('crs', {}).get('properties', {}).get('name') + elif any(file.name.endswith(suffix) for suffix in RASTER_FILETYPES): + cog_path = get_cog_path(file) + if cog_path: + cog_set.append(dict(name=file.name, path=cog_path)) + elif not any(file.name.endswith(suffix) for suffix in IGNORE_FILETYPES): + print('\t\tUnable to convert', file.name) + + if combine: + # combine only works for vector data currently, assumes consistent projection + all_features = [] + for geodata in geodata_set: + all_features += geodata.get('features') + geodata_set = [dict(name=file_item.name, features=all_features)] + + for geodata in geodata_set: + data, features = geodata.get('data'), geodata.get('features') + if data is None and len(features): + gdf = geopandas.GeoDataFrame.from_features(features) + gdf = gdf.set_crs(source_projection, allow_override=True) + gdf = gdf.to_crs(4326) + data = json.loads(gdf.to_json()) + properties = {} + for feature in data.get('features'): + for name, value in feature.get('properties', {}).items(): + if name not in properties: + properties[name] = [] + if value not in properties[name]: + properties[name].append(value) + metadata['properties'] = properties + vector_data = VectorData.objects.create( + name=geodata.get('name'), + dataset=file_item.dataset, + source_file=file_item, + metadata=metadata, + ) + vector_data.write_geojson_data(data) + print('\t\t', str(vector_data), 'created for ' + geodata.get('name')) + + for cog in cog_set: + cog_path = cog.get('path') + source = large_image.open(cog_path) + metadata.update(source.getMetadata()) + raster_data = RasterData.objects.create( + name=cog.get('name'), + dataset=file_item.dataset, + source_file=file_item, + metadata=metadata, + ) + with open(cog_path, 'rb') as f: + raster_data.cloud_optimized_geotiff.save(cog_path.name, ContentFile(f.read())) + print('\t\t', str(raster_data), 'created for ' + cog.get('name')) + + +def convert_file_item(file_item): + # write contents to temporary directory for conversion + with tempfile.TemporaryDirectory() as temp_dir: + if file_item.file_type == 'zip': + archive_path = Path(temp_dir, 'archive.zip') + with open(archive_path, 'wb') as archive_file: + archive_file.write(file_item.file.open('rb').read()) + with zipfile.ZipFile(archive_path) as zip_archive: + files = [] + for file in zip_archive.infolist(): + if not file.is_dir(): + filepath = Path(temp_dir, Path(file.filename).name) + with open(filepath, 'wb') as f: + f.write(zip_archive.open(file).read()) + files.append(filepath) + combine = file_item.metadata.get('combine_contents', False) + convert_files(*files, file_item=file_item, combine=combine) + else: + filepath = Path(temp_dir, file_item.name) + with open(filepath, 'wb') as f: + with file_item.file.open('rb') as contents: + f.write(contents.read()) + convert_files(filepath, file_item=file_item) diff --git a/uvdat/core/tasks/data.py b/uvdat/core/tasks/data.py new file mode 100644 index 00000000..9ec2c610 --- /dev/null +++ b/uvdat/core/tasks/data.py @@ -0,0 +1,23 @@ +import json + +from django.contrib.gis.geos import GEOSGeometry + +from uvdat.core.models import VectorData, VectorFeature + + +def create_vector_features(vector_data: VectorData): + features = vector_data.read_geojson_data()['features'] + vector_features = [] + for feature in features: + vector_features.append( + VectorFeature( + vector_data=vector_data, + geometry=GEOSGeometry(json.dumps(feature['geometry'])), + properties=feature['properties'], + ) + ) + + created = VectorFeature.objects.bulk_create(vector_features) + print('\t\t', f'{len(created)} vector features created.') + + return created diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index b7db2ce2..ff863841 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -1,17 +1,104 @@ from celery import shared_task -from uvdat.core.models import Dataset, FileItem, RasterMapLayer, SourceRegion, VectorMapLayer -from uvdat.core.tasks.map_layers import save_vector_features +from uvdat.core.models import Dataset, FileItem, Layer, LayerFrame, RasterData, VectorData -from .map_layers import create_raster_map_layer, create_vector_map_layer +from .conversion import convert_file_item +from .data import create_vector_features from .networks import create_network from .regions import create_source_regions +def create_layers_and_frames(dataset, layer_options=None): + Layer.objects.filter(dataset=dataset).delete() + LayerFrame.objects.filter(layer__dataset=dataset).delete() + vectors = VectorData.objects.filter(dataset=dataset) + rasters = RasterData.objects.filter(dataset=dataset) + + if layer_options is None: + layer_options = [ + dict(name=data.name.split('.')[0].replace('_', ' '), frames=None, data=data.name) + for data in [*vectors.all(), *rasters.all()] + ] + + for layer_info in layer_options: + layer = Layer.objects.create( + dataset=dataset, + name=layer_info.get('name', dataset.name), + metadata=layer_info.get('metadata', {}), + ) + frames = layer_info.get('frames') + if frames is None: + frames = [] + kwargs = dict(dataset=dataset) + data_name = layer_info.get('data') + source_file_name = layer_info.get('source_file') + if data_name is not None: + kwargs['name'] = data_name + if source_file_name is not None: + kwargs['source_file__name'] = source_file_name + + for layer_data in [ + *VectorData.objects.filter(**kwargs).order_by('name').all(), + *RasterData.objects.filter(**kwargs).order_by('name').all(), + ]: + frame_property = layer_info.get('frame_property') + additional_filters = layer_info.get('additional_filters', {}) + metadata = layer_data.metadata or {} + properties = metadata.get('properties') + bands = metadata.get('bands') + if properties and frame_property and frame_property in properties: + for value in properties[frame_property]: + frames.append( + dict( + name=value, + index=len(frames), + data=layer_data.name, + source_filters={frame_property: value, **additional_filters}, + ) + ) + elif bands and len(bands) > 1: + for band in bands: + frames.append( + dict( + name=band, + index=len(frames), + data=layer_data.name, + source_filters=dict(band=band), + ) + ) + else: + frames.append( + dict( + name=layer_data.name, + index=len(frames), + data=layer_data.name, + ) + ) + for i, frame_info in enumerate(frames): + index = frame_info.get('index', i) + data_name = frame_info.get('data') + if data_name: + vector, raster = None, None + vectors = VectorData.objects.filter(dataset=dataset, name=data_name) + rasters = RasterData.objects.filter(dataset=dataset, name=data_name) + if vectors.count(): + vector = vectors.first() + if rasters.count(): + raster = rasters.first() + LayerFrame.objects.create( + name=frame_info.get('name', f'Frame {index}'), + layer=layer, + index=index, + vector=vector, + raster=raster, + source_filters=frame_info.get('source_filters', dict(band=1)), + ) + + @shared_task def convert_dataset( dataset_id, - style_options=None, + layer_options=None, network_options=None, region_options=None, ): @@ -19,31 +106,24 @@ def convert_dataset( dataset.processing = True dataset.save() - if dataset.dataset_type == dataset.DatasetType.RASTER: - RasterMapLayer.objects.filter(dataset=dataset).delete() - for file_to_convert in FileItem.objects.filter(dataset=dataset): - create_raster_map_layer( - file_to_convert, - style_options=style_options, - ) - - elif dataset.dataset_type == dataset.DatasetType.VECTOR: - VectorMapLayer.objects.filter(dataset=dataset).delete() - SourceRegion.objects.filter(dataset=dataset).delete() - - for file_to_convert in FileItem.objects.filter(dataset=dataset): - vector_map_layer = create_vector_map_layer( - file_to_convert, - style_options=style_options, - ) - if network_options: - create_network(vector_map_layer, network_options) - elif region_options: - create_source_regions(vector_map_layer, region_options) - - # Create vector tiles after geojson_data may have - # been altered by create_network or create_source_regions - save_vector_features(vector_map_layer=vector_map_layer) + VectorData.objects.filter(dataset=dataset).delete() + RasterData.objects.filter(dataset=dataset).delete() + + for file_to_convert in FileItem.objects.filter(dataset=dataset): + convert_file_item(file_to_convert) + + vectors = VectorData.objects.filter(dataset=dataset) + for vector_data in vectors.all(): + if network_options: + create_network(vector_data, network_options) + elif region_options: + create_source_regions(vector_data, region_options) + + # Create vector features after geojson_data may have + # been altered by create_network or create_source_regions + create_vector_features(vector_data) + + create_layers_and_frames(dataset, layer_options) dataset.processing = False dataset.save() diff --git a/uvdat/core/tasks/map_layers.py b/uvdat/core/tasks/map_layers.py deleted file mode 100644 index 00d2d7ab..00000000 --- a/uvdat/core/tasks/map_layers.py +++ /dev/null @@ -1,235 +0,0 @@ -import json -from pathlib import Path -import tempfile -import zipfile - -from django.contrib.gis.geos import GEOSGeometry -from django.core.files.base import ContentFile -import geopandas -import numpy -import rasterio -import shapefile -from webcolors import name_to_hex - -from uvdat.core.models import RasterMapLayer, VectorMapLayer -from uvdat.core.models.map_layers import VectorFeature - - -def add_styling(geojson_data, style_options): - if not style_options: - return geojson_data - - outline = style_options.get('outline') - palette = style_options.get('palette') - color_property = style_options.get('color_property') - color_delimiter = style_options.get('color_delimiter', ',') - - features = [] - for index, feature in enumerate(geojson_data.iterfeatures()): - feature_colors = [] - if color_property: - color_value = feature['properties'].get(color_property) - if color_value: - feature_colors += str(color_value).split(color_delimiter) - - if isinstance(palette, dict): - feature_colors = [palette[c] for c in feature_colors if c in palette] - elif isinstance(palette, list): - feature_colors.append(palette[index % len(palette)]) - - if outline: - feature_colors.append(outline) - - feature_colors = [name_to_hex(c) for c in feature_colors] - feature['properties']['colors'] = ','.join(feature_colors) - features.append(feature) - return geopandas.GeoDataFrame.from_features(features) - - -def rasterio_convert_raster( - map_layer, temp_dir, raw_data_path, transparency_threshold, trim_distribution_percentage -): - raster_path = Path(temp_dir, 'raster.tiff') - with open(raw_data_path, 'rb') as raw_data: - # read original data with rasterio - input_data = rasterio.open(raw_data) - output_data = rasterio.open( - raster_path, - 'w', - driver='GTiff', - height=input_data.height, - width=input_data.width, - count=1, - dtype=numpy.float32, - crs=input_data.crs, - transform=input_data.transform, - ) - band = input_data.read(1) - - # alter data according to style options - if trim_distribution_percentage: - # trim a number of values from both ends of the distribution - histogram, bin_edges = numpy.histogram(band, bins=1000) - trim_n = band.size * trim_distribution_percentage - new_min = None - new_max = None - sum_values = 0 - for bin_index, bin_count in enumerate(histogram): - bin_edge = bin_edges[bin_index] - sum_values += bin_count - if new_min is None and sum_values > trim_n: - new_min = bin_edge - if new_max is None and sum_values > band.size - trim_n: - new_max = bin_edge - if new_min: - band[band < new_min] = new_min - if new_max: - band[band > new_max] = new_max - if transparency_threshold is not None: - # clamp values below transparency threshold - band[band < transparency_threshold] = transparency_threshold - - band_range = [float(band.min()), float(band.max())] - map_layer.default_style['data_range'] = band_range - - output_data.write(band, 1) - output_data.close() - return raster_path - - -def create_raster_map_layer(file_item, style_options): - """Save a RasterMapLayer from a FileItem's contents.""" - import large_image - import large_image_converter - - # create new raster map layer object - new_map_layer = RasterMapLayer.objects.create( - dataset=file_item.dataset, - metadata={}, - default_style=style_options, - index=file_item.index, - ) - print('\t', f'RasterMapLayer {new_map_layer.id} created.') - - transparency_threshold = style_options.get('transparency_threshold') - trim_distribution_percentage = style_options.get('trim_distribution_percentage') - - with tempfile.TemporaryDirectory() as temp_dir: - raster_path = None - raw_data_path = Path(temp_dir, 'raw_data.tiff') - cog_raster_path = Path(temp_dir, 'cog_raster.tiff') - # write original data from file field to file in temp_dir - with open(raw_data_path, 'wb') as raw_data: - with file_item.file.open('rb') as raw_data_archive: - raw_data.write(raw_data_archive.read()) - - if transparency_threshold or trim_distribution_percentage: - # if data needs to be altered according to style options, use rasterio - raster_path = rasterio_convert_raster( - new_map_layer, - temp_dir, - raw_data_path, - transparency_threshold, - trim_distribution_percentage, - ) - else: - try: - # if applicable, use large_image to interpret data - source = large_image.open(raw_data_path) - if source.geospatial: - raster_path = raw_data_path - min_val, max_val = None, None - source._scanForMinMax(numpy.int64) - for _frame, range_spec in source._bandRanges.items(): - frame_min = numpy.min(range_spec.get('min', numpy.empty(1))) - frame_max = numpy.max(range_spec.get('max', numpy.empty(1))) - if min_val is None or frame_min < min_val: - min_val = frame_min - if max_val is None or frame_max < max_val: - max_val = frame_max - new_map_layer.default_style['data_range'] = [int(min_val), int(max_val)] - except large_image.exceptions.TileSourceError: - # if original data cannot be interpreted by large_image, use rasterio - raster_path = rasterio_convert_raster( - new_map_layer, - temp_dir, - raw_data_path, - transparency_threshold, - trim_distribution_percentage, - ) - - # use large_image to convert new raster data to COG - large_image_converter.convert(str(raster_path), str(cog_raster_path)) - with open(cog_raster_path, 'rb') as cog_raster_file: - # save COG to new raster map layer - new_map_layer.cloud_optimized_geotiff.save( - cog_raster_path, ContentFile(cog_raster_file.read()) - ) - - return new_map_layer - - -def create_vector_map_layer(file_item, style_options): - """Save a VectorMapLayer from a FileItem's contents.""" - new_map_layer = VectorMapLayer.objects.create( - dataset=file_item.dataset, - metadata={}, - default_style=style_options, - index=file_item.index, - ) - print('\t', f'VectorMapLayer {new_map_layer.id} created.') - - if file_item.file_type == 'zip': - geojson_data = convert_zip_to_geojson(file_item) - elif file_item.file_type == 'geojson' or file_item.file_type == 'json': - source_data = json.load(file_item.file.open()) - source_projection = source_data.get('crs', {}).get('properties', {}).get('name') - geojson_data = geopandas.GeoDataFrame.from_features(source_data.get('features')) - if source_projection: - geojson_data = geojson_data.set_crs(source_projection) - geojson_data = geojson_data.to_crs(4326) - - geojson_data = add_styling(geojson_data, style_options) - new_map_layer.write_geojson_data(geojson_data.to_json()) - new_map_layer.save() - - return new_map_layer - - -def convert_zip_to_geojson(file_item): - features = [] - source_projection = None - with tempfile.TemporaryDirectory() as temp_dir: - archive_path = Path(temp_dir, 'archive.zip') - with open(archive_path, 'wb') as archive_file: - archive_file.write(file_item.file.open('rb').read()) - with zipfile.ZipFile(archive_path) as zip_archive: - filenames = zip_archive.namelist() - for filename in filenames: - if filename.endswith('.shp'): - sf = shapefile.Reader(f'{archive_path}/{filename}') - features.extend(sf.__geo_interface__['features']) - if filename.endswith('.prj'): - source_projection = zip_archive.open(filename).read().decode() - geodata = geopandas.GeoDataFrame.from_features(features) - geodata = geodata.set_crs(source_projection, allow_override=True) - geodata = geodata.to_crs(4326) - return geodata - - -def save_vector_features(vector_map_layer: VectorMapLayer): - features = vector_map_layer.read_geojson_data()['features'] - vector_features = [] - for feature in features: - vector_features.append( - VectorFeature( - map_layer=vector_map_layer, - geometry=GEOSGeometry(json.dumps(feature['geometry'])), - properties=feature['properties'], - ) - ) - - created = VectorFeature.objects.bulk_create(vector_features) - print('\t', f'{len(created)} vector features created.') - - return created diff --git a/uvdat/core/tasks/networks.py b/uvdat/core/tasks/networks.py index ddf78561..b981f58c 100644 --- a/uvdat/core/tasks/networks.py +++ b/uvdat/core/tasks/networks.py @@ -6,7 +6,7 @@ import numpy import shapely -from uvdat.core.models import Network, NetworkEdge, NetworkNode, VectorFeature, VectorMapLayer +from uvdat.core.models import Network, NetworkEdge, NetworkNode, VectorFeature NODE_RECOVERY_MODES = [ 'random', @@ -20,21 +20,23 @@ ] -def create_network(vector_map_layer, network_options): +def create_network(vector_data, network_options): # Overwrite previous results - dataset = vector_map_layer.dataset - Network.objects.filter(dataset=dataset).delete() + dataset = vector_data.dataset + Network.objects.filter(vector_data__dataset=dataset).delete() network = Network.objects.create( - dataset=dataset, category=dataset.category, metadata={'source': 'Parsed from GeoJSON.'} + category=dataset.category, + vector_data=vector_data, + metadata={'source': 'Parsed from GeoJSON.'}, ) connection_column = network_options.get('connection_column') connection_column_delimiter = network_options.get('connection_column_delimiter') node_id_column = network_options.get('node_id_column') - source_data = vector_map_layer.read_geojson_data() + source_data = vector_data.read_geojson_data() geodata = geopandas.GeoDataFrame.from_features(source_data.get('features')).set_crs(4326) - geodata[connection_column].fillna('', inplace=True) + geodata.fillna({'connection_column': ''}, inplace=True) edge_set = geodata[geodata.geom_type != 'Point'] node_set = geodata[geodata.geom_type == 'Point'] @@ -167,17 +169,19 @@ def create_network(vector_map_layer, network_options): line_geometry=edge_line_geometry, metadata=metadata, ) - # rewrite vector_map_layer geojson_data with updated features - vector_map_layer.write_geojson_data(geojson_from_network(vector_map_layer.dataset)) - vector_map_layer.metadata['network'] = True - vector_map_layer.save() + + all_nodes = NetworkNode.objects.filter(network=network) + all_edges = NetworkEdge.objects.filter(network=network) + print('\t\t', f'{all_nodes.count()} nodes and {all_edges.count()} edges created.') + # rewrite vector_data geojson_data with updated features + vector_data.write_geojson_data(geojson_from_network(vector_data.dataset)) + vector_data.metadata['network'] = True + vector_data.save() def geojson_from_network(dataset): - total_nodes = 0 - total_edges = 0 new_feature_set = [] - for n in NetworkNode.objects.filter(network__dataset=dataset): + for n in NetworkNode.objects.filter(network__vector_data__dataset=dataset): node_as_feature = { 'id': n.id, 'type': 'Feature', @@ -188,9 +192,8 @@ def geojson_from_network(dataset): 'properties': dict(node_id=n.id, **n.metadata), } new_feature_set.append(node_as_feature) - total_nodes += 1 - for e in NetworkEdge.objects.filter(network__dataset=dataset): + for e in NetworkEdge.objects.filter(network__vector_data__dataset=dataset): edge_as_feature = { 'id': e.id, 'type': 'Feature', @@ -206,19 +209,17 @@ def geojson_from_network(dataset): ), } new_feature_set.append(edge_as_feature) - total_edges += 1 new_geodata = geopandas.GeoDataFrame.from_features(new_feature_set) - print('\t', f'GeoJSON feature set created for {total_nodes} nodes and {total_edges} edges.') return new_geodata.to_json() def create_vector_features_from_network(network): - map_layer, _ = VectorMapLayer.objects.get_or_create(dataset=network.dataset, index=0) + vector_data = network.vector_data VectorFeature.objects.bulk_create( [ VectorFeature( - map_layer=map_layer, + vector_data=vector_data, geometry=node.location, properties=dict(node_id=node.id, **node.metadata), ) @@ -228,7 +229,7 @@ def create_vector_features_from_network(network): VectorFeature.objects.bulk_create( [ VectorFeature( - map_layer=map_layer, + vector_data=vector_data, geometry=edge.line_geometry, properties=dict( edge_id=edge.id, diff --git a/uvdat/core/tasks/osmnx.py b/uvdat/core/tasks/osmnx.py index 02f72a9b..d18dc4c1 100644 --- a/uvdat/core/tasks/osmnx.py +++ b/uvdat/core/tasks/osmnx.py @@ -2,8 +2,8 @@ from django.contrib.gis.geos import LineString, Point import osmnx -from uvdat.core.models import Dataset, Network, NetworkEdge, NetworkNode, Project, VectorMapLayer -from uvdat.core.tasks.map_layers import save_vector_features +from uvdat.core.models import Dataset, Network, NetworkEdge, NetworkNode, Project, VectorData +from uvdat.core.tasks.data import create_vector_features from uvdat.core.tasks.networks import geojson_from_network @@ -11,14 +11,13 @@ def get_or_create_road_dataset(project, location): dataset, created = Dataset.objects.get_or_create( name=f'{location} Road Network', description='Roads and intersections retrieved from OpenStreetMap via OSMnx', - dataset_type=Dataset.DatasetType.VECTOR, category='transportation', ) if created: project.datasets.add(dataset) print('Clearing previous results...') - Network.objects.filter(dataset=dataset).delete() + Network.objects.filter(vector_data__dataset=dataset).delete() return dataset @@ -34,8 +33,11 @@ def metadata_for_row(row): def load_roads(project_id, location): project = Project.objects.get(id=project_id) dataset = get_or_create_road_dataset(project, location) + vector_data = VectorData.objects.create(dataset=dataset) network = Network.objects.create( - dataset=dataset, category='roads', metadata={'source': 'Fetched with OSMnx.'} + category='roads', + vector_data=vector_data, + metadata={'source': 'Fetched with OSMnx.'}, ) print(f'Fetching road data for {location}...') @@ -86,11 +88,6 @@ def load_roads(project_id, location): edge.metadata = metadata_for_row(edge_data) edge.save() - vector_map_layer = VectorMapLayer.objects.create( - dataset=dataset, - metadata=dict(network=True), - index=0, - ) - vector_map_layer.write_geojson_data(geojson_from_network(dataset)) - save_vector_features(vector_map_layer) + vector_data.write_geojson_data(geojson_from_network(dataset)) + create_vector_features(vector_data) print('Done.') diff --git a/uvdat/core/tasks/regions.py b/uvdat/core/tasks/regions.py index 1b3d8fe5..676d0ab7 100644 --- a/uvdat/core/tasks/regions.py +++ b/uvdat/core/tasks/regions.py @@ -3,12 +3,16 @@ from django.contrib.gis.geos import GEOSGeometry import geopandas -from uvdat.core.models import SourceRegion +from uvdat.core.models import Region -def create_source_regions(vector_map_layer, region_options): +def create_source_regions(vector_data, region_options): + # Overwrite previous results + dataset = vector_data.dataset + Region.objects.filter(dataset=dataset).delete() + name_property = region_options.get('name_property') - geodata = vector_map_layer.read_geojson_data() + geodata = vector_data.read_geojson_data() region_count = 0 new_feature_set = [] @@ -27,11 +31,11 @@ def create_source_regions(vector_map_layer, region_options): geometry['coordinates'] = [geometry['coordinates']] # Create region with properties and MultiPolygon - region = SourceRegion( + region = Region( name=name, boundary=GEOSGeometry(str(geometry)), metadata=properties, - dataset=vector_map_layer.dataset, + dataset=dataset, ) region.save() region_count += 1 @@ -51,6 +55,6 @@ def create_source_regions(vector_map_layer, region_options): # Save updated features to layer new_geodata = geopandas.GeoDataFrame.from_features(new_feature_set).set_crs(3857) new_geodata.to_crs(4326) - vector_map_layer.write_geojson_data(new_geodata.to_json()) - vector_map_layer.save() - print('\t', f'{region_count} regions created.') + vector_data.write_geojson_data(new_geodata.to_json()) + vector_data.save() + print('\t\t', f'{region_count} regions created.') diff --git a/uvdat/core/tasks/simulations.py b/uvdat/core/tasks/simulations.py index 91dcc5f1..1e6b7c49 100644 --- a/uvdat/core/tasks/simulations.py +++ b/uvdat/core/tasks/simulations.py @@ -43,13 +43,13 @@ def get_network_node_elevations(network_nodes, elevation_data): @shared_task def flood_scenario_1(simulation_result_id, network, elevation_data, flood_area): - from uvdat.core.models import Network, RasterMapLayer, SimulationResult, VectorMapLayer + from uvdat.core.models import Network, RasterData, SimulationResult, VectorData result = SimulationResult.objects.get(id=simulation_result_id) try: network = Network.objects.get(id=network) - elevation_data = RasterMapLayer.objects.get(id=elevation_data) - flood_area = VectorMapLayer.objects.get(id=flood_area) + elevation_data = RasterData.objects.get(id=elevation_data) + flood_area = VectorData.objects.get(id=flood_area) except Exception: result.error_message = 'Object not found.' result.save() @@ -127,11 +127,11 @@ def recovery_scenario(simulation_result_id, node_failure_simulation_result, reco def segment_curbs(simulation_result_id, imagery_layer): from tile2net import Raster - from uvdat.core.models import Dataset, FileItem, RasterMapLayer, SimulationResult + from uvdat.core.models import Dataset, FileItem, RasterData, SimulationResult result = SimulationResult.objects.get(id=simulation_result_id) try: - imagery_layer = RasterMapLayer.objects.get(id=imagery_layer) + imagery_layer = RasterData.objects.get(id=imagery_layer) except Exception: result.error_message = 'Object not found.' result.save() @@ -189,7 +189,6 @@ def segment_curbs(simulation_result_id, imagery_layer): name=dataset_name, description='Segmentation generated by tile2net from orthoimagery', category='segmentation', - dataset_type=Dataset.DatasetType.VECTOR, metadata={'creation_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, ) result.project.datasets.add(dataset) diff --git a/uvdat/core/tests/factories.py b/uvdat/core/tests/factories.py index e0165e6c..4a5d6c69 100644 --- a/uvdat/core/tests/factories.py +++ b/uvdat/core/tests/factories.py @@ -6,8 +6,7 @@ from factory.faker import faker import factory.fuzzy -from uvdat.core.models import Dataset, Project -from uvdat.core.models.map_layers import RasterMapLayer, VectorMapLayer +from uvdat.core.models import Dataset, Layer, LayerFrame, Project, RasterData, VectorData from uvdat.core.models.networks import Network, NetworkEdge, NetworkNode @@ -57,7 +56,6 @@ class Meta: model = Dataset name = factory.Faker('name') - dataset_type = Dataset.DatasetType.VECTOR category = factory.Faker( 'random_element', elements=[ @@ -71,11 +69,53 @@ class Meta: ) +class RasterDataFactory(factory.django.DjangoModelFactory): + class Meta: + model = RasterData + + name = factory.Faker('name') + dataset = factory.SubFactory(DatasetFactory) + cloud_optimized_geotiff = factory.django.FileField( + filename=factory.Faker('file_name', extension='tif'), + from_path=Path(__file__).parent / 'data' / 'sample_cog.tif', + ) + + +class VectorDataFactory(factory.django.DjangoModelFactory): + class Meta: + model = VectorData + + name = factory.Faker('name') + dataset = factory.SubFactory(DatasetFactory) + geojson_data = factory.django.FileField( + filename=factory.Faker('file_name', extension='json'), + from_path=Path(__file__).parent / 'data' / 'sample_geo.json', + ) + + +class LayerFactory(factory.django.DjangoModelFactory): + class Meta: + model = Layer + + name = factory.Faker('name') + dataset = factory.SubFactory(DatasetFactory) + + +class LayerFrameFactory(factory.django.DjangoModelFactory): + class Meta: + model = LayerFrame + + name = factory.Faker('name') + layer = factory.SubFactory(LayerFactory) + vector = factory.SubFactory(VectorDataFactory) + raster = factory.SubFactory(RasterDataFactory) + + class NetworkFactory(factory.django.DjangoModelFactory): class Meta: model = Network - dataset = factory.SubFactory(DatasetFactory) + vector_data = factory.SubFactory(VectorDataFactory) class NetworkNodeFactory(factory.django.DjangoModelFactory): @@ -102,25 +142,3 @@ class Meta: @factory.lazy_attribute def line_geometry(self): return LineString(self.from_node.location, self.to_node.location) - - -class RasterMapLayerFactory(factory.django.DjangoModelFactory): - class Meta: - model = RasterMapLayer - - dataset = factory.SubFactory(DatasetFactory) - cloud_optimized_geotiff = factory.django.FileField( - filename=factory.Faker('file_name', extension='tif'), - from_path=Path(__file__).parent / 'data' / 'sample_cog.tif', - ) - - -class VectorMapLayerFactory(factory.django.DjangoModelFactory): - class Meta: - model = VectorMapLayer - - dataset = factory.SubFactory(DatasetFactory) - geojson_file = factory.django.FileField( - filename=factory.Faker('file_name', extension='json'), - from_path=Path(__file__).parent / 'data' / 'sample_geo.json', - ) diff --git a/uvdat/core/tests/factory_fixtures.py b/uvdat/core/tests/factory_fixtures.py index 2b2f0d32..2ea9ff73 100644 --- a/uvdat/core/tests/factory_fixtures.py +++ b/uvdat/core/tests/factory_fixtures.py @@ -4,14 +4,16 @@ from .factories import ( DatasetFactory, + LayerFactory, + LayerFrameFactory, NetworkEdgeFactory, NetworkFactory, NetworkNodeFactory, ProjectFactory, - RasterMapLayerFactory, + RasterDataFactory, SuperUserFactory, UserFactory, - VectorMapLayerFactory, + VectorDataFactory, ) @@ -94,23 +96,45 @@ def network_edge(network_edge_factory): return network_edge_factory() -# Raster Map Layer +# Raster Data @pytest.fixture -def raster_map_layer_factory(): - return RasterMapLayerFactory +def raster_data_factory(): + return RasterDataFactory @pytest.fixture -def raster_map_layer(raster_map_layer_factory): - return raster_map_layer_factory() +def raster_data(raster_data_factory): + return raster_data_factory() -# Vector Map Layer +# Vector Data @pytest.fixture -def vector_map_layer_factory(): - return VectorMapLayerFactory +def vector_data_factory(): + return VectorDataFactory @pytest.fixture -def vector_map_layer(vector_map_layer_factory): - return vector_map_layer_factory() +def vector_data(vector_data_factory): + return vector_data_factory() + + +# Layer +@pytest.fixture +def layer_factory(): + return LayerFactory + + +@pytest.fixture +def layer(layer_factory): + return layer_factory() + + +# Layer Frame +@pytest.fixture +def layer_frame_factory(): + return LayerFrameFactory + + +@pytest.fixture +def layer_frame(layer_frame_factory): + return layer_frame_factory() diff --git a/uvdat/core/tests/test_dataset.py b/uvdat/core/tests/test_dataset.py index 67552814..d806606f 100644 --- a/uvdat/core/tests/test_dataset.py +++ b/uvdat/core/tests/test_dataset.py @@ -35,7 +35,7 @@ def test_rest_dataset_gcc_no_networks(authenticated_api_client, dataset: Dataset def test_rest_dataset_gcc_empty_network( authenticated_api_client, project: Project, network: Network ): - dataset = network.dataset + dataset = network.vector_data.dataset project.datasets.add(dataset) resp = authenticated_api_client.get( f'/api/v1/datasets/{dataset.id}/gcc/?project={project.id}&exclude_nodes=1' @@ -78,7 +78,7 @@ def test_rest_dataset_gcc( # | # * - dataset = network.dataset + dataset = network.vector_data.dataset project.datasets.add(dataset) resp = authenticated_api_client.get( f'/api/v1/datasets/{dataset.id}/gcc/' @@ -90,58 +90,47 @@ def test_rest_dataset_gcc( assert sorted(resp.json()) == sorted([n.id for n in larger_group]) -@pytest.mark.parametrize('dataset_type', [x[0] for x in Dataset.DatasetType.choices]) @pytest.mark.django_db -def test_rest_dataset_map_layers_incorrect_layer_type( +def test_rest_dataset_layers( authenticated_api_client, dataset_factory, - raster_map_layer_factory, - vector_map_layer_factory, - dataset_type, + layer_factory, ): - dataset = dataset_factory(dataset_type=dataset_type) + dataset = dataset_factory() + layers = [layer_factory(dataset=dataset) for _ in range(3)] - # Intentionally create the wrong map layer type - factory = ( - vector_map_layer_factory - if dataset_type == Dataset.DatasetType.RASTER - else raster_map_layer_factory - ) - for _ in range(3): - factory(dataset=dataset) - - # Check that nothing is returned, since map_layers will only - # return map layers that match the dataset's type - resp = authenticated_api_client.get(f'/api/v1/datasets/{dataset.id}/map_layers/') + resp = authenticated_api_client.get(f'/api/v1/datasets/{dataset.id}/layers/') assert resp.status_code == 200 - assert resp.json() == [] + + data: list[dict] = resp.json() + assert len(data) == 3 + + # Assert these lists are the same objects + assert sorted([x['id'] for x in data]) == sorted([x.id for x in layers]) -@pytest.mark.parametrize('dataset_type', [x[0] for x in Dataset.DatasetType.choices]) @pytest.mark.django_db -def test_rest_dataset_map_layers( +def test_rest_dataset_data_objects( authenticated_api_client, dataset_factory, - raster_map_layer_factory, - vector_map_layer_factory, - dataset_type, + vector_data_factory, + raster_data_factory, ): - dataset = dataset_factory(dataset_type=dataset_type) - factory = ( - vector_map_layer_factory - if dataset_type == Dataset.DatasetType.VECTOR - else raster_map_layer_factory - ) - map_layers = [factory(dataset=dataset) for _ in range(3)] + dataset = dataset_factory() + data_objects = [ + *[vector_data_factory(dataset=dataset) for _ in range(3)], + *[raster_data_factory(dataset=dataset) for _ in range(3)], + ] - resp = authenticated_api_client.get(f'/api/v1/datasets/{dataset.id}/map_layers/') + resp = authenticated_api_client.get(f'/api/v1/datasets/{dataset.id}/data/') assert resp.status_code == 200 data: list[dict] = resp.json() - assert len(data) == 3 + assert len(data) == 6 + print(data) # Assert these lists are the same objects - assert sorted([x['id'] for x in data]) == sorted([x.id for x in map_layers]) + assert sorted([x['id'] for x in data]) == sorted([x.id for x in data_objects]) @pytest.mark.django_db @@ -154,7 +143,7 @@ def test_rest_dataset_network_no_network(authenticated_api_client, dataset: Data @pytest.mark.django_db def test_rest_dataset_network(authenticated_api_client, network_edge): network = network_edge.network - dataset = network.dataset + dataset = network.vector_data.dataset assert network_edge.from_node != network_edge.to_node resp = authenticated_api_client.get(f'/api/v1/datasets/{dataset.id}/network/') diff --git a/uvdat/core/tests/test_populate.py b/uvdat/core/tests/test_populate.py index d58807ba..dc886ed6 100644 --- a/uvdat/core/tests/test_populate.py +++ b/uvdat/core/tests/test_populate.py @@ -6,15 +6,17 @@ Chart, Dataset, FileItem, + Layer, + LayerFrame, Network, NetworkEdge, NetworkNode, Project, - RasterMapLayer, + RasterData, + Region, SimulationResult, - SourceRegion, + VectorData, VectorFeature, - VectorMapLayer, ) @@ -27,9 +29,9 @@ def test_populate(): # smaller subset for faster evaluation # 0 is MBTA Rapid Transit, tests network eval # 4 is Massachusetts Elevation Data, tests raster eval - # 5 is Boston Neighborhoods, tests regions eval - # 8 is Boston Sea Level Rises, tests multi-map-layer dataset eval - dataset_indexes = [0, 4, 5, 8] + # 6 is Boston Neighborhoods, tests regions eval + # 9 is Boston Projected Flood Events, tests multilayer/multiframe dataset eval + dataset_indexes = [0, 4, 6, 9] call_command( 'populate', @@ -41,12 +43,14 @@ def test_populate(): assert Chart.objects.all().count() == 1 assert Project.objects.all().count() == 2 assert Dataset.objects.all().count() == 4 - assert FileItem.objects.all().count() == 7 + assert FileItem.objects.all().count() == 13 + assert Layer.objects.all().count() == 6 + assert LayerFrame.objects.all().count() == 12 assert Network.objects.all().count() == 1 assert NetworkEdge.objects.all().count() == 164 assert NetworkNode.objects.all().count() == 158 - assert RasterMapLayer.objects.all().count() == 1 + assert RasterData.objects.all().count() == 1 assert SimulationResult.objects.all().count() == 0 - assert SourceRegion.objects.all().count() == 24 - assert VectorMapLayer.objects.all().count() == 5 - assert VectorFeature.objects.count() == 351 + assert Region.objects.all().count() == 24 + assert VectorData.objects.all().count() == 11 + assert VectorFeature.objects.count() == 357 diff --git a/uvdat/settings.py b/uvdat/settings.py index 8899facb..bdad7b2d 100644 --- a/uvdat/settings.py +++ b/uvdat/settings.py @@ -50,6 +50,7 @@ def mutate_configuration(configuration: ComposedConfiguration) -> None: configuration.AUTHENTICATION_BACKENDS += ['guardian.backends.ObjectPermissionBackend'] configuration.REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] += [ 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', + 'rest_framework.authentication.TokenAuthentication', ] configuration.REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = [ 'rest_framework.permissions.IsAuthenticated' diff --git a/uvdat/urls.py b/uvdat/urls.py index aa6293f1..74d62716 100644 --- a/uvdat/urls.py +++ b/uvdat/urls.py @@ -5,16 +5,19 @@ from drf_yasg import openapi from drf_yasg.views import get_schema_view from rest_framework import permissions, routers +from rest_framework.authtoken.views import obtain_auth_token from uvdat.core.rest import ( ChartViewSet, DatasetViewSet, + LayerFrameViewSet, + LayerViewSet, ProjectViewSet, - RasterMapLayerViewSet, + RasterDataViewSet, + RegionViewSet, SimulationViewSet, - SourceRegionViewSet, UserViewSet, - VectorMapLayerViewSet, + VectorDataViewSet, ) router = routers.SimpleRouter() @@ -29,9 +32,11 @@ router.register(r'projects', ProjectViewSet, basename='projects') router.register(r'datasets', DatasetViewSet, basename='datasets') router.register(r'charts', ChartViewSet, basename='charts') -router.register(r'rasters', RasterMapLayerViewSet, basename='rasters') -router.register(r'vectors', VectorMapLayerViewSet, basename='vectors') -router.register(r'source-regions', SourceRegionViewSet, basename='source-regions') +router.register(r'layers', LayerViewSet, basename='layers') +router.register(r'layer-frames', LayerFrameViewSet, basename='layer-frames') +router.register(r'rasters', RasterDataViewSet, basename='rasters') +router.register(r'vectors', VectorDataViewSet, basename='vectors') +router.register(r'source-regions', RegionViewSet, basename='source-regions') router.register(r'simulations', SimulationViewSet, basename='simulations') urlpatterns = [ @@ -42,6 +47,7 @@ path('api/v1/', include(router.urls)), path('api/docs/redoc/', schema_view.with_ui('redoc'), name='docs-redoc'), path('api/docs/swagger/', schema_view.with_ui('swagger'), name='docs-swagger'), + path('api/v1/token/', obtain_auth_token), # Redirect all other server requests to Vue client path('', RedirectView.as_view(url=settings.HOMEPAGE_REDIRECT_URL)), # type: ignore ]