Skip to content

Commit

Permalink
Merge pull request #126 from A-Baji/multi-table-insert
Browse files Browse the repository at this point in the history 
Multi table insert
  • Loading branch information
@jverswijver
jverswijver authored Aug 31, 2022
2 parents c3f5853 + 4745fe6 commit c17ee2c
Show file tree
Hide file tree
Showing 6 changed files with 685 additions and 57 deletions.
2 changes: 1 addition & 1 deletion docker-compose-dev.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ services:
command: pharus
fakeservices.datajoint.io:
<<: *net
image: datajoint/nginx:v0.0.18
image: datajoint/nginx:v0.2.2
environment:
- ADD_pharus_TYPE=REST
- ADD_pharus_ENDPOINT=pharus:5000
Expand Down
210 changes: 165 additions & 45 deletions pharus/component_interface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import types
import io
import numpy as np
from functools import reduce


class NumpyEncoder(json.JSONEncoder):
Expand Down Expand Up @@ -44,9 +45,7 @@ def dumps(cls, obj):
return json.dumps(obj, cls=cls)


class QueryComponent:
attributes_route_format = None

class FetchComponent:
def __init__(self, name, component_config, static_config, jwt_payload: dict):
lcls = locals()
self.name = name
Expand All @@ -64,10 +63,6 @@ def __init__(self, name, component_config, static_config, jwt_payload: dict):
self.route = component_config["route"]
exec(component_config["dj_query"], globals(), lcls)
self.dj_query = lcls["dj_query"]
if self.attributes_route_format:
self.attribute_route = self.attributes_route_format.format(
route=component_config["route"]
)
if "restriction" in component_config:
exec(component_config["restriction"], globals(), lcls)
self.dj_restriction = lcls["restriction"]
Expand Down Expand Up @@ -114,8 +109,164 @@ def restriction(self):
]
)

def dj_query_route(self):
fetch_metadata = self.fetch_metadata
record_header, table_records, total_count = _DJConnector._fetch_records(
query=fetch_metadata["query"] & self.restriction,
fetch_args=fetch_metadata["fetch_args"],
)
return dict(
recordHeader=record_header, records=table_records, totalCount=total_count
)


class InsertComponent:
fields_route_format = "{route}/fields"

def __init__(
self, name, component_config, static_config, payload, jwt_payload: dict
):
self.name = name
self.payload = payload
if static_config:
self.static_variables = types.MappingProxyType(static_config)
if not all(k in component_config for k in ("x", "y", "height", "width")):
self.mode = "dynamic"
else:
self.mode = "fixed"
self.x = component_config["x"]
self.y = component_config["y"]
self.height = component_config["height"]
self.width = component_config["width"]
self.type = component_config["type"]
self.route = component_config["route"]
self.connection = dj.conn(
host=jwt_payload["databaseAddress"],
user=jwt_payload["username"],
password=jwt_payload["password"],
reset=True,
)
self.fields_map = component_config.get("map")
self.tables = [
getattr(
dj.VirtualModule(
s,
s,
connection=self.connection,
),
t,
)
for s, t in (_.split(".") for _ in component_config["tables"])
]
self.parents = sorted(
set(
[
p
for t in self.tables
for p in t.parents(as_objects=True)
if p.full_table_name not in (t.full_table_name for t in self.tables)
]
),
key=lambda p: p.full_table_name,
)

def dj_query_route(self):
with self.connection.transaction:
destination_lookup = reduce(
lambda m0, m1: dict(
m0,
**(
{
m_t["input"]
if "input" in m_t
else m_t["destination"]: m_t["destination"]
for m_t in m1["map"]
}
if m1["type"] == "table"
else {
m1["input"]
if "input" in m1
else m1["destination"]: m1["destination"]
}
),
),
self.fields_map or [],
{},
)
for t in self.tables:
t.insert(
[
{
a: v
for k, v in r.items()
if (a := destination_lookup.get(k, k))
in t.heading.attributes
}
for r in self.payload["submissions"]
]
)
return "Insert successful"

def fields_route(self):
parent_attributes = sorted(set(sum([p.primary_key for p in self.parents], [])))
source_fields = {
**{
(p_name := f"{p.database}.{dj.utils.to_camel_case(p.table_name)}"): {
"values": p.fetch("KEY"),
"type": "table",
"name": p_name,
}
for p in self.parents
},
**{
a: {"datatype": v.type, "type": "attribute", "name": v.name}
for t in self.tables
for a, v in t.heading.attributes.items()
if a not in parent_attributes
},
}

if not self.fields_map:
return dict(fields=list(source_fields.values()))
return dict(
fields=[
dict(
(field := source_fields.pop(m["destination"])),
name=m["input" if "input" in m else "destination"],
**(
{
"values": field["values"]
if "map" not in m
else [
{
input_lookup[k]: v
for k, v in r.items()
if k
in (
input_lookup := {
table_m["destination"]: table_m[
"input"
if "input" in table_m
else "destination"
]
for table_m in m["map"]
}
)
}
for r in field["values"]
]
}
if m["type"] == "table"
else {}
),
)
for m in self.fields_map
]
+ list(source_fields.values())
)


class TableComponent(QueryComponent):
class TableComponent(FetchComponent):
attributes_route_format = "{route}/attributes"

def __init__(self, *args, **kwargs):
Expand Down Expand Up @@ -235,7 +386,7 @@ def dj_query_route(self):
)


class PlotPlotlyStoredjsonComponent(QueryComponent):
class PlotPlotlyStoredjsonComponent(FetchComponent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.frontend_map = {
Expand Down Expand Up @@ -264,39 +415,7 @@ def dj_query_route(self):
)


class BasicQuery(QueryComponent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.frontend_map = {
"source": "sci-viz/src/Components/Plots/FullPlotly.tsx",
"target": "FullPlotly",
}
self.response_examples = {
"dj_query_route": {
"recordHeader": ["subject_uuid", "session_start_time", "session_uuid"],
"records": [
[
"00778394-c956-408d-8a6c-ca3b05a611d5",
1565436299.0,
"fb9bdf18-76be-452b-ac4e-21d5de3a6f9f",
]
],
"totalCount": 1,
},
}

def dj_query_route(self):
fetch_metadata = self.fetch_metadata
record_header, table_records, total_count = _DJConnector._fetch_records(
query=fetch_metadata["query"] & self.restriction,
fetch_args=fetch_metadata["fetch_args"],
)
return dict(
recordHeader=record_header, records=table_records, totalCount=total_count
)


class FileImageAttachComponent(QueryComponent):
class FileImageAttachComponent(FetchComponent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.frontend_map = {
Expand All @@ -319,11 +438,12 @@ def dj_query_route(self):


type_map = {
"basicquery": BasicQuery,
"basicquery": FetchComponent,
"plot:plotly:stored_json": PlotPlotlyStoredjsonComponent,
"table": TableComponent,
"metadata": MetadataComponent,
"file:image:attach": FileImageAttachComponent,
"slider": BasicQuery,
"dropdown-query": BasicQuery,
"slider": FetchComponent,
"dropdown-query": FetchComponent,
"form": InsertComponent,
}
45 changes: 37 additions & 8 deletions pharus/dynamic_api_gen.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import json
import re

from pharus.component_interface import InsertComponent, TableComponent


def populate_api():
header_template = """# Auto-generated rest api
Expand All @@ -24,25 +26,26 @@ def populate_api():
"""
route_template = """
@app.route('{route}', methods=['GET'])
@app.route('{route}', methods=['{rest_verb}'])
@protected_route
def {method_name}(jwt_payload: dict) -> dict:
if request.method in {{'GET'}}:
if request.method in ['{rest_verb}']:
try:
component_instance = type_map['{component_type}'](name='{component_name}',
component_config={component},
static_config={static_config},
jwt_payload=jwt_payload)
jwt_payload=jwt_payload,
{payload})
return component_instance.{method_name_type}()
except Exception as e:
return traceback.format_exc(), 500
"""
route_template_nologin = """
@app.route('{route}', methods=['GET'])
@app.route('{route}', methods=['{rest_verb}'])
def {method_name}() -> dict:
if request.method in {{'GET'}}:
if request.method in ['{rest_verb}']:
jwt_payload = dict(
databaseAddress=os.environ["PHARUS_HOST"],
username=os.environ["PHARUS_USER"],
Expand All @@ -52,7 +55,8 @@ def {method_name}() -> dict:
component_instance = type_map['{component_type}'](name='{component_name}',
component_config={component},
static_config={static_config},
jwt_payload=jwt_payload)
jwt_payload=jwt_payload,
{payload})
return component_instance.{method_name_type}()
except Exception as e:
return traceback.format_exc(), 500
Expand Down Expand Up @@ -99,11 +103,13 @@ def {method_name}() -> dict:
f.write(
(active_route_template).format(
route=grid["route"],
rest_verb="GET",
method_name=grid["route"].replace("/", ""),
component_type="basicquery",
component_name="dynamicgrid",
component=json.dumps(grid),
static_config=static_config,
payload="",
method_name_type="dj_query_route",
)
)
Expand All @@ -114,32 +120,55 @@ def {method_name}() -> dict:
else grid["components"]
).items():
if re.match(
r"^(table|metadata|plot|file|slider|dropdown-query).*$",
r"^(table|metadata|plot|file|slider|dropdown-query|form).*$",
comp["type"],
):
f.write(
(active_route_template).format(
route=comp["route"],
rest_verb="POST" if comp["type"] == "form" else "GET",
method_name=comp["route"].replace("/", ""),
component_type=comp["type"],
component_name=comp_name,
component=json.dumps(comp),
static_config=static_config,
payload="payload=request.get_json()"
if comp["type"] == "form"
else "",
method_name_type="dj_query_route",
)
)
if type_map[comp["type"]].attributes_route_format:
if issubclass(type_map[comp["type"]], InsertComponent):
fields_route = type_map[
comp["type"]
].fields_route_format.format(route=comp["route"])
f.write(
(active_route_template).format(
route=fields_route,
rest_verb="GET",
method_name=fields_route.replace("/", ""),
component_type=comp["type"],
component_name=comp_name,
component=json.dumps(comp),
static_config=static_config,
payload="payload=None",
method_name_type="fields_route",
)
)
elif issubclass(type_map[comp["type"]], TableComponent):
attributes_route = type_map[
comp["type"]
].attributes_route_format.format(route=comp["route"])
f.write(
(active_route_template).format(
route=attributes_route,
rest_verb="GET",
method_name=attributes_route.replace("/", ""),
component_type=comp["type"],
component_name=comp_name,
component=json.dumps(comp),
static_config=static_config,
payload="",
method_name_type="attributes_route",
)
)
Loading

0 comments on commit c17ee2c

Please sign in to comment.