Skip to content

Commit

Permalink
fix: function names as per pep8
Browse files Browse the repository at this point in the history
  • Loading branch information
sanjibansg committed Oct 16, 2023
1 parent 75e05d4 commit 110a327
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 45 deletions.
47 changes: 24 additions & 23 deletions api/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,17 @@
import jwt
import substrait_validator as sv
from duckdb import DuckDBPyConnection
from fastapi import Depends, FastAPI, File, Form, HTTPException, UploadFile, status
from fastapi import (Depends, FastAPI, File, Form, HTTPException, UploadFile,
status)
from fastapi.middleware.trustedhost import TrustedHostMiddleware
from fastapi.openapi.utils import get_openapi
from fastapi.security import OAuth2PasswordBearer
from fastapi_health import health
from loguru import logger

from backend.duckdb import (
CheckDuckDBConnection,
DeleteTableFromDuckDB,
DuckDBConnection,
ExecuteDuckDb,
ParseFromDuckDB,
)
from backend.duckdb import (DuckDBConnection, check_duckdb_connection,
delete_table_from_duckDB, execute_duckdb,
parse_from_duckDB)
from backend.ttl_cache import TTL_Cache

app = FastAPI()
Expand Down Expand Up @@ -62,25 +59,26 @@ def ping():


@app.on_event("startup")
async def Initialize():
async def initialize():
app.state.duck_pool = DuckDBConnection()
app.state.schema_cache = TTL_Cache(
maxsize=100,
ttl=3600,
on_expire=lambda key, _: DeleteTableFromDuckDB(key, get_duck_conn()),
on_expire=lambda key, _: delete_table_from_duckDB(
key, get_duck_conn()),
)


@app.get("/health/duckcb/")
def CheckBackendConn(conn):
CheckDuckDBConnection(conn)
def check_backend_conn(conn):
check_duckdb_connection(conn)


app.add_api_route("/health", health([CheckBackendConn]))
app.add_api_route("/health", health([check_backend_conn]))


@app.post("/validate/", status_code=status.HTTP_200_OK)
async def Validate(plan: dict, override_levels: list[int]):
async def validate(plan: dict, override_levels: list[int]):
try:
logger.info("Validating plan using substrait-validator!")
config = sv.Config()
Expand All @@ -90,12 +88,14 @@ async def Validate(plan: dict, override_levels: list[int]):
logger.info("Plan validated successfully!")
except Exception as e:
raise HTTPException(
status_code=500, detail="Substrait Validator Internal Error: " + str(e)
status_code=500, detail="Substrait Validator Internal Error: "
+ str(e)
)


@app.post("/validate/file/", status_code=status.HTTP_200_OK)
async def ValidateFile(file: UploadFile = File(), override_levels: list[int] = Form()):
async def validate_file(file: UploadFile = File(),
override_levels: list[int] = Form()):
try:
logger.info("Validating file using substrait-validator!")
config = sv.Config()
Expand All @@ -106,12 +106,13 @@ async def ValidateFile(file: UploadFile = File(), override_levels: list[int] = F
logger.info("File validated successfully!")
except Exception as e:
raise HTTPException(
status_code=500, detail="Substrait Validator Internal Error: " + str(e)
status_code=500, detail="Substrait Validator Internal Error: "
+ str(e)
)


@app.post("/add_schema/")
def AddSchema(
def add_schema(
data: dict,
headers: dict = Depends(verify_token),
db_conn: DuckDBPyConnection = Depends(get_duck_conn),
Expand All @@ -132,23 +133,23 @@ def AddSchema(
query = query[:-2]
query += ");"

response = ExecuteDuckDb(query, db_conn)
response = execute_duckdb(query, db_conn)
app.state.schema_cache[table_name] = None
return response


@app.post("/parse/", status_code=status.HTTP_200_OK)
def ParseToSubstrait(
def parse_to_substrait(
data: dict,
headers: dict = Depends(verify_token),
db_conn: DuckDBPyConnection = Depends(get_duck_conn),
):
response = ParseFromDuckDB(data.get("query"), db_conn)
response = parse_from_duckDB(data.get("query"), db_conn)
return response


# For defining custom documentation for the server
def SubstraitFiddleOpenAPI():
def substrait_fiddle_openapi():
if app.openapi_schema:
return app.openapi_schema
openapi_schema = get_openapi(
Expand All @@ -161,4 +162,4 @@ def SubstraitFiddleOpenAPI():
return app.openapi_schema


app.openapi = SubstraitFiddleOpenAPI
app.openapi = substrait_fiddle_openapi
41 changes: 21 additions & 20 deletions api/backend/duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@ def __init__(self):
self.conn_pool = []

query_lineitem = """CREATE TABLE IF NOT EXISTS lineitem(
l_orderkey INTEGER NOT NULL,
l_partkey INTEGER NOT NULL,
l_suppkey INTEGER NOT NULL,
l_linenumber INTEGER NOT NULL,
l_quantity INTEGER NOT NULL,
l_extendedprice DECIMAL(15,2) NOT NULL,
l_discount DECIMAL(15,2) NOT NULL,
l_tax DECIMAL(15,2) NOT NULL,
l_returnflag VARCHAR NOT NULL,
l_linestatus VARCHAR NOT NULL,
l_shipdate DATE NOT NULL,
l_commitdate DATE NOT NULL,
l_receiptdate DATE NOT NULL,
l_shipinstruct VARCHAR NOT NULL,
l_shipmode VARCHAR NOT NULL,
l_orderkey INTEGER NOT NULL,
l_partkey INTEGER NOT NULL,
l_suppkey INTEGER NOT NULL,
l_linenumber INTEGER NOT NULL,
l_quantity INTEGER NOT NULL,
l_extendedprice DECIMAL(15,2) NOT NULL,
l_discount DECIMAL(15,2) NOT NULL,
l_tax DECIMAL(15,2) NOT NULL,
l_returnflag VARCHAR NOT NULL,
l_linestatus VARCHAR NOT NULL,
l_shipdate DATE NOT NULL,
l_commitdate DATE NOT NULL,
l_receiptdate DATE NOT NULL,
l_shipinstruct VARCHAR NOT NULL,
l_shipmode VARCHAR NOT NULL,
l_comment VARCHAR NOT NULL);"""
conn = duckdb.connect("duck.db")
conn.execute(query=query_lineitem)
Expand All @@ -47,7 +47,7 @@ def initialize(self):
return con


def CheckDuckDBConnection(con):
def check_duckdb_connection(con):
status = {"db_health": "unavailable"}
try:
con.execute(query="SHOW TABLES;").fetchall()
Expand All @@ -62,7 +62,7 @@ def CheckDuckDBConnection(con):
return status


def ExecuteDuckDb(query, con):
def execute_duckdb(query, con):
try:
con.execute(query=query)
return {"message": "DuckDB Operation successful"}
Expand All @@ -74,19 +74,20 @@ def ExecuteDuckDb(query, con):
)


def DeleteTableFromDuckDB(table_name, con):
def delete_table_from_duckDB(table_name, con):
try:
con.execute(query="DROP TABLE " + table_name + ";")
except Exception as e:
logger.error(e)


def ParseFromDuckDB(query, con):
def parse_from_duckDB(query, con):
try:
result = con.get_substrait_json(query).fetchone()[0]
return result
except Exception as e:
raise HTTPException(
status_code=500,
detail="Substrait DuckDB Internal Error while parsing SQL Query: " + str(e),
detail="Substrait DuckDB Internal Error while parsing SQL Query: "
+ str(e),
)
7 changes: 5 additions & 2 deletions api/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ def test_validate_json():

def test_validate_binary():
with TestClient(app) as client:
url = "https://github.com/westonpace/substrait-viewer/blob/main/demo/q1.bin"
url = (
"https://github.com/westonpace/"
"substrait-viewer/blob/main/demo/q1.bin"
)
response = requests.get(url)
file_content = response.content

Expand All @@ -52,7 +55,7 @@ def test_duckdb_execute():
res = client.post(
"/execute/duckdb/",
json={
"query": """CREATE TABLE IF NOT EXISTS
"query": """CREATE TABLE IF NOT EXISTS
weather(city VARCHAR, temp_lo INTEGER);""",
},
)
Expand Down

0 comments on commit 110a327

Please sign in to comment.