-
Notifications
You must be signed in to change notification settings - Fork 4.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for types in BigQuery #232
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,6 +15,30 @@ | |
|
||
from redash.utils import JSONEncoder | ||
|
||
def transform_row(row, fields): | ||
column_index = 0 | ||
row_data = {} | ||
|
||
for cell in row["f"]: | ||
field = fields[column_index] | ||
cell_value = cell['v'] | ||
|
||
if cell_value is None: | ||
pass | ||
# Otherwise just cast the value | ||
elif field['type'] == 'INTEGER': | ||
cell_value = int(cell_value) | ||
|
||
elif field['type'] == 'FLOAT': | ||
cell_value = float(cell_value) | ||
|
||
elif field['type'] == 'BOOLEAN': | ||
cell_value = cell_value in ('True', 'true', 'TRUE') | ||
|
||
row_data[field["name"]] = cell_value | ||
column_index += 1 | ||
|
||
return row_data | ||
|
||
def bigquery(connection_string): | ||
def load_key(filename): | ||
|
@@ -67,28 +91,21 @@ def query_runner(query): | |
query_reply = get_query_results(jobs, project_id=project_id, | ||
job_id=insert_response['jobReference']['jobId'], start_index=current_row) | ||
|
||
logging.debug("bigquery replied: %s", query_reply) | ||
|
||
rows = [] | ||
field_names = [] | ||
for f in query_reply["schema"]["fields"]: | ||
field_names.append(f["name"]) | ||
|
||
while ("rows" in query_reply) and current_row < query_reply['totalRows']: | ||
for row in query_reply["rows"]: | ||
row_data = {} | ||
column_index = 0 | ||
for cell in row["f"]: | ||
row_data[field_names[column_index]] = cell["v"] | ||
column_index += 1 | ||
|
||
rows.append(row_data) | ||
rows.append(transform_row(row, query_reply["schema"]["fields"])) | ||
|
||
current_row += len(query_reply['rows']) | ||
query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'], | ||
startIndex=current_row).execute() | ||
|
||
columns = [{'name': name, | ||
'friendly_name': name, | ||
'type': None} for name in field_names] | ||
columns = [{'name': f["name"], | ||
'friendly_name': f["name"], | ||
'type': None} for f in query_reply["schema"]["fields"]] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If you populate the 'type' field of a column using the type information BigQuery provides. It will help the UI with knowing how to better render stuff (like float vs. integer or datetime vs only date). Currently there are no consts for this, so see here the list of types: https://github.com/EverythingMe/redash/blob/master/redash/data/query_runner_pg.py#L17 |
||
|
||
data = { | ||
"columns": columns, | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You might want to add support for date/time fields too, otherwise you will have issues with charts.